diff --git a/admin_api/admin_api.go b/admin_api/admin_api.go index 24fa8a3..04a7030 100644 --- a/admin_api/admin_api.go +++ b/admin_api/admin_api.go @@ -14,15 +14,52 @@ import ( var log = logging.MustGetLogger("cursorius-server") func createSchema(db database.Database) (graphql.Schema, error) { - credentialType := graphql.NewObject(graphql.ObjectConfig{ - Name: "Credential", + secretType := graphql.NewObject(graphql.ObjectConfig{ + Name: "Secret", + Description: "A secret available for use inside of a pipeline.", + Fields: graphql.Fields{ + "id": &graphql.Field{ + Type: graphql.NewNonNull(graphql.String), + Description: "The id of the secret.", + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + if secret, ok := p.Source.(database.Secret); ok { + return secret.Id, nil + } + return nil, nil + }, + }, + "name": &graphql.Field{ + Type: graphql.NewNonNull(graphql.String), + Description: "The name of the secret.", + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + if secret, ok := p.Source.(database.Secret); ok { + return secret.Name, nil + } + return nil, nil + }, + }, + "secret": &graphql.Field{ + Type: graphql.NewNonNull(graphql.String), + Description: "The secret.", + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + if secret, ok := p.Source.(database.Secret); ok { + return secret.Secret, nil + } + return nil, nil + }, + }, + }, + }) + + cloneCredentialType := graphql.NewObject(graphql.ObjectConfig{ + Name: "CloneCredential", Description: "A credential for authenticating with the pipeline source host.", Fields: graphql.Fields{ "id": &graphql.Field{ Type: graphql.NewNonNull(graphql.String), Description: "The id of the credential.", Resolve: func(p graphql.ResolveParams) (interface{}, error) { - if credential, ok := p.Source.(database.Credential); ok { + if credential, ok := p.Source.(database.CloneCredential); ok { return credential.Id, nil } return nil, nil @@ -32,7 +69,7 @@ func createSchema(db database.Database) (graphql.Schema, error) { Type: graphql.NewNonNull(graphql.String), Description: "The name of the credential.", Resolve: func(p graphql.ResolveParams) (interface{}, error) { - if credential, ok := p.Source.(database.Credential); ok { + if credential, ok := p.Source.(database.CloneCredential); ok { return credential.Name, nil } return nil, nil @@ -42,7 +79,7 @@ func createSchema(db database.Database) (graphql.Schema, error) { Type: graphql.NewNonNull(graphql.String), Description: "The credential type.", Resolve: func(p graphql.ResolveParams) (interface{}, error) { - if credential, ok := p.Source.(database.Credential); ok { + if credential, ok := p.Source.(database.CloneCredential); ok { return credential.Type, nil } return nil, nil @@ -52,7 +89,7 @@ func createSchema(db database.Database) (graphql.Schema, error) { Type: graphql.NewNonNull(graphql.String), Description: "The username to user with the credential.", Resolve: func(p graphql.ResolveParams) (interface{}, error) { - if credential, ok := p.Source.(database.Credential); ok { + if credential, ok := p.Source.(database.CloneCredential); ok { return credential.Username, nil } return nil, nil @@ -62,7 +99,7 @@ func createSchema(db database.Database) (graphql.Schema, error) { Type: graphql.NewNonNull(graphql.String), Description: "The secret for the credential.", Resolve: func(p graphql.ResolveParams) (interface{}, error) { - if credential, ok := p.Source.(database.Credential); ok { + if credential, ok := p.Source.(database.CloneCredential); ok { return credential.Secret, nil } return nil, nil @@ -210,16 +247,28 @@ func createSchema(db database.Database) (graphql.Schema, error) { return nil, nil }, }, - "credentialId": &graphql.Field{ - Type: graphql.String, + "cloneCredential": &graphql.Field{ + Type: cloneCredentialType, Description: "The configured credential for cloning the pipeline source.", Resolve: func(p graphql.ResolveParams) (interface{}, error) { if pipeline, ok := p.Source.(database.Pipeline); ok { - return pipeline.Credential, nil + if pipeline.CloneCredential != nil { + return db.GetCloneCredentialById(*pipeline.CloneCredential) + } } return nil, nil }, }, + "secrets": &graphql.Field{ + Type: graphql.NewNonNull(graphql.NewList(graphql.NewNonNull(secretType))), + Description: "The list of secrets for the pipeline.", + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + if pipeline, ok := p.Source.(database.Pipeline); ok { + return db.GetSecretsForPipeline(pipeline.Id) + } + return []database.Secret{}, nil + }, + }, "webhooks": &graphql.Field{ Type: graphql.NewNonNull(graphql.NewList(graphql.NewNonNull(webhookType))), Description: "The list of webhooks for the pipeline.", @@ -269,8 +318,8 @@ func createSchema(db database.Database) (graphql.Schema, error) { return db.GetPipelines() }, }, - "Credential": &graphql.Field{ - Type: credentialType, + "CloneCredential": &graphql.Field{ + Type: cloneCredentialType, Args: graphql.FieldConfigArgument{ "id": &graphql.ArgumentConfig{ Type: graphql.NewNonNull(graphql.String), @@ -282,16 +331,23 @@ func createSchema(db database.Database) (graphql.Schema, error) { if err != nil { return nil, err } - return db.GetCredentialById(id) + return db.GetCloneCredentialById(id) }, }, - "Credentials": &graphql.Field{ - Type: graphql.NewNonNull(graphql.NewList(credentialType)), + "CloneCredentials": &graphql.Field{ + Type: graphql.NewNonNull(graphql.NewList(cloneCredentialType)), Args: graphql.FieldConfigArgument{}, Resolve: func(p graphql.ResolveParams) (interface{}, error) { return db.GetCredentials() }, }, + "Secrets": &graphql.Field{ + Type: graphql.NewNonNull(graphql.NewList(secretType)), + Args: graphql.FieldConfigArgument{}, + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + return db.GetSecrets() + }, + }, }, }) @@ -311,7 +367,7 @@ func createSchema(db database.Database) (graphql.Schema, error) { "pollInterval": &graphql.ArgumentConfig{ Type: graphql.Int, }, - "credentialId": &graphql.ArgumentConfig{ + "cloneCredentialId": &graphql.ArgumentConfig{ Type: graphql.String, }, }, @@ -324,7 +380,7 @@ func createSchema(db database.Database) (graphql.Schema, error) { } var credential *uuid.UUID - if credentialVal, ok := params.Args["credentialId"]; ok { + if credentialVal, ok := params.Args["cloneCredentialId"]; ok { id, err := uuid.Parse(credentialVal.(string)) if err != nil { return nil, err @@ -375,9 +431,9 @@ func createSchema(db database.Database) (graphql.Schema, error) { return webhook, nil }, }, - "createCredential": &graphql.Field{ - Type: credentialType, - Description: "Create a new credential", + "createCloneCredential": &graphql.Field{ + Type: cloneCredentialType, + Description: "Create a new CloneCredential", Args: graphql.FieldConfigArgument{ "name": &graphql.ArgumentConfig{ Type: graphql.NewNonNull(graphql.String), @@ -396,7 +452,7 @@ func createSchema(db database.Database) (graphql.Schema, error) { credential, err := db.CreateCredential( params.Args["name"].(string), - database.CredentialType(params.Args["type"].(string)), + database.CloneCredentialType(params.Args["type"].(string)), params.Args["username"].(string), params.Args["secret"].(string), ) @@ -406,11 +462,34 @@ func createSchema(db database.Database) (graphql.Schema, error) { return credential, nil }, }, - "setPipelineCredential": &graphql.Field{ - Type: pipelineType, - Description: "Add an credential to a pipeline", + "createSecret": &graphql.Field{ + Type: secretType, + Description: "Create a new secret", Args: graphql.FieldConfigArgument{ - "credentialId": &graphql.ArgumentConfig{ + "name": &graphql.ArgumentConfig{ + Type: graphql.NewNonNull(graphql.String), + }, + "secret": &graphql.ArgumentConfig{ + Type: graphql.NewNonNull(graphql.String), + }, + }, + Resolve: func(params graphql.ResolveParams) (interface{}, error) { + + secret, err := db.CreateSecret( + params.Args["name"].(string), + params.Args["secret"].(string), + ) + if err != nil { + return nil, err + } + return secret, nil + }, + }, + "setPipelineCloneCredential": &graphql.Field{ + Type: pipelineType, + Description: "Set the CloneCredential used by a pipeline to clone the source repo", + Args: graphql.FieldConfigArgument{ + "cloneCredentialId": &graphql.ArgumentConfig{ Type: graphql.String, }, "pipelineId": &graphql.ArgumentConfig{ @@ -424,19 +503,19 @@ func createSchema(db database.Database) (graphql.Schema, error) { return nil, err } - if credentialIdVal, ok := params.Args["credentialId"]; ok { - credentialId, err := uuid.Parse(credentialIdVal.(string)) + if cloneCredentialIdVal, ok := params.Args["cloneCredentialId"]; ok { + cloneCredentialId, err := uuid.Parse(cloneCredentialIdVal.(string)) if err != nil { return nil, err } - pipeline, err := db.SetPipelineCredential(pipelineId, &credentialId) + pipeline, err := db.SetPipelineCloneCredential(pipelineId, &cloneCredentialId) if err != nil { return nil, err } return pipeline, nil } else { - pipeline, err := db.SetPipelineCredential(pipelineId, nil) + pipeline, err := db.SetPipelineCloneCredential(pipelineId, nil) if err != nil { return nil, err } @@ -445,6 +524,76 @@ func createSchema(db database.Database) (graphql.Schema, error) { }, }, + "addSecretToPipeline": &graphql.Field{ + Type: pipelineType, + Description: "Allow a secret to be accessed by a pipeline.", + Args: graphql.FieldConfigArgument{ + "secretId": &graphql.ArgumentConfig{ + Type: graphql.String, + }, + "pipelineId": &graphql.ArgumentConfig{ + Type: graphql.NewNonNull(graphql.String), + }, + }, + Resolve: func(params graphql.ResolveParams) (interface{}, error) { + + secretId, err := uuid.Parse(params.Args["secretId"].(string)) + if err != nil { + return nil, err + } + + pipelineId, err := uuid.Parse(params.Args["pipelineId"].(string)) + if err != nil { + return nil, err + } + + err = db.AssignSecretToPipeline(pipelineId, secretId) + if err != nil { + return nil, err + } + + pipeline, err := db.GetPipelineById(pipelineId) + if err != nil { + return nil, err + } + return pipeline, nil + }, + }, + "removeSecretFromPipeline": &graphql.Field{ + Type: pipelineType, + Description: "Remove a pipeline's access to a secret.", + Args: graphql.FieldConfigArgument{ + "secretId": &graphql.ArgumentConfig{ + Type: graphql.String, + }, + "pipelineId": &graphql.ArgumentConfig{ + Type: graphql.NewNonNull(graphql.String), + }, + }, + Resolve: func(params graphql.ResolveParams) (interface{}, error) { + + secretId, err := uuid.Parse(params.Args["secretId"].(string)) + if err != nil { + return nil, err + } + + pipelineId, err := uuid.Parse(params.Args["pipelineId"].(string)) + if err != nil { + return nil, err + } + + err = db.RemoveSecretFromPipeline(pipelineId, secretId) + if err != nil { + return nil, err + } + + pipeline, err := db.GetPipelineById(pipelineId) + if err != nil { + return nil, err + } + return pipeline, nil + }, + }, }, }) diff --git a/database/db.go b/database/db.go index 6eb0b7b..f484f41 100644 --- a/database/db.go +++ b/database/db.go @@ -95,7 +95,7 @@ CREATE TABLE version ( ); -CREATE TABLE credentials ( +CREATE TABLE clone_credentials ( id UUID PRIMARY KEY, name TEXT NOT NULL, type TEXT NOT NULL, @@ -104,15 +104,34 @@ CREATE TABLE credentials ( ); CREATE TABLE pipelines ( - id UUID PRIMARY KEY, - name TEXT NOT NULL, - url TEXT NOT NULL, - poll_interval INTEGER, - credential UUID DEFAULT NULL, + id UUID PRIMARY KEY, + name TEXT NOT NULL, + url TEXT NOT NULL, + poll_interval INTEGER, + clone_credential UUID DEFAULT NULL, - CONSTRAINT fk_credential - FOREIGN KEY(credential) - REFERENCES credentials(id) + CONSTRAINT fk_clone_credential + FOREIGN KEY(clone_credential) + REFERENCES clone_credentials(id) +); + +CREATE TABLE secrets ( + id UUID PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + secret TEXT NOT NULL +); + +CREATE TABLE pipeline_secret_mappings ( + pipeline UUID NOT NULL, + secret UUID NOT NULL, + + CONSTRAINT fk_pipeline + FOREIGN KEY(pipeline) + REFERENCES pipelines(id), + + CONSTRAINT fk_secret + FOREIGN KEY(secret) + REFERENCES secrets(id) ); CREATE TABLE webhooks ( diff --git a/database/func.go b/database/func.go index fce2489..9232e1e 100644 --- a/database/func.go +++ b/database/func.go @@ -10,7 +10,7 @@ import ( func (db *Database) GetPipelines() ([]Pipeline, error) { query := ` -SELECT id, name, url, poll_interval, credential +SELECT id, name, url, poll_interval, clone_credential FROM pipelines;` pipelines := make([]Pipeline, 0) @@ -24,7 +24,7 @@ FROM pipelines;` for rows.Next() { var pipeline Pipeline var idStr string - if err := rows.Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.Credential); err != nil { + if err := rows.Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.CloneCredential); err != nil { return pipelines, err } @@ -58,7 +58,7 @@ WHERE id=$1;` func (db *Database) CreatePipeline(name string, url string, pollInterval int, credential *uuid.UUID) (Pipeline, error) { query := ` -INSERT INTO pipelines (id, name, url, poll_interval, credential) +INSERT INTO pipelines (id, name, url, poll_interval, clone_credential) VALUES (uuid_generate_v4(), $1, $2, $3, $4) RETURNING id, name, url, poll_interval;` @@ -77,12 +77,12 @@ RETURNING id, name, url, poll_interval;` return pipeline, nil } -func (db *Database) SetPipelineCredential(pipelineId uuid.UUID, credentialId *uuid.UUID) (Pipeline, error) { +func (db *Database) SetPipelineCloneCredential(pipelineId uuid.UUID, credentialId *uuid.UUID) (Pipeline, error) { query := ` UPDATE pipelines -SET credential=$1 +SET clone_credential=$1 WHERE id=$2 -RETURNING name, url, poll_interval, credential;` +RETURNING name, url, poll_interval, clone_credential;` pipeline := Pipeline{ Id: pipelineId, @@ -90,7 +90,7 @@ RETURNING name, url, poll_interval, credential;` err := db.Conn.QueryRow(context.Background(), query, credentialId, pipelineId).Scan( - &pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.Credential, + &pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.CloneCredential, ) if err != nil { return pipeline, fmt.Errorf("Could not add credential to pipeline: %w", err) @@ -104,7 +104,7 @@ func (db *Database) RemovePipelineCredential(pipelineId uuid.UUID) (Pipeline, er UPDATE pipelines SET credential=null WHERE id=$1 -RETURNING name, url, poll_interval, credential;` +RETURNING name, url, poll_interval, clone_credential;` pipeline := Pipeline{ Id: pipelineId, @@ -112,7 +112,7 @@ RETURNING name, url, poll_interval, credential;` err := db.Conn.QueryRow(context.Background(), query, pipelineId).Scan( - &pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.Credential, + &pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.CloneCredential, ) if err != nil { return pipeline, fmt.Errorf("Could not add credential to pipeline: %w", err) @@ -194,13 +194,13 @@ RETURNING id, server_type, secret, pipeline;` return webhook, nil } -func (db *Database) CreateCredential(name string, credentialtype CredentialType, username string, secret string) (Credential, error) { +func (db *Database) CreateCredential(name string, credentialtype CloneCredentialType, username string, secret string) (CloneCredential, error) { query := ` -INSERT INTO credentials (id, name, type, username, secret) +INSERT INTO clone_credentials (id, name, type, username, secret) VALUES(uuid_generate_v4(), $1, $2, $3, $4) RETURNING id, name, type, username, secret;` - credential := Credential{} + credential := CloneCredential{} var idStr string err := db.Conn.QueryRow( context.Background(), @@ -224,15 +224,15 @@ RETURNING id, name, type, username, secret;` return credential, nil } -func (db *Database) GetCredentialById(id uuid.UUID) (Credential, error) { +func (db *Database) GetCloneCredentialById(id uuid.UUID) (CloneCredential, error) { query := ` SELECT name, type, username, secret -FROM credentials +FROM clone_credentials WHERE id=$1;` log.Debugf("requested credential with id %v", id) - credential := Credential{ + credential := CloneCredential{ Id: id, } @@ -244,12 +244,12 @@ WHERE id=$1;` return credential, nil } -func (db *Database) GetCredentials() ([]Credential, error) { +func (db *Database) GetCredentials() ([]CloneCredential, error) { query := ` SELECT id, name, type, username, secret -FROM credentials;` +FROM clone_credentials;` - credentials := make([]Credential, 0) + credentials := make([]CloneCredential, 0) rows, err := db.Conn.Query(context.Background(), query) if err != nil { @@ -258,7 +258,7 @@ FROM credentials;` defer rows.Close() for rows.Next() { - var credential Credential + var credential CloneCredential var idStr string if err := rows.Scan(&idStr, &credential.Name, &credential.Type, &credential.Username, &credential.Secret); err != nil { return credentials, err @@ -395,3 +395,128 @@ func (db *Database) UpdatePipelineRefs(pipelineId uuid.UUID, refsMap map[string] log.Debugf("copyCount: %v", copyCount) return nil } + +func (db *Database) GetSecrets() ([]Secret, error) { + query := ` +SELECT id, name, secret +FROM secrets;` + + secrets := make([]Secret, 0) + + rows, err := db.Conn.Query(context.Background(), query) + if err != nil { + return secrets, fmt.Errorf("Could not query database for secrets: %w", err) + } + defer rows.Close() + + for rows.Next() { + var secret Secret + var idStr string + if err := rows.Scan(&idStr, &secret.Name, &secret.Secret); err != nil { + return secrets, err + } + + secret.Id, err = uuid.Parse(idStr) + if err != nil { + return secrets, err + } + secrets = append(secrets, secret) + } + + return secrets, nil +} + +func (db *Database) GetSecretById(id uuid.UUID) (Secret, error) { + query := ` +SELECT id, name, secret +FROM secrets +WHERE id=$1;` + + secret := Secret{ + Id: id, + } + + err := db.Conn.QueryRow(context.Background(), query, id).Scan(&secret.Name, &secret.Secret) + if err != nil { + return secret, fmt.Errorf("Could not query database for secret with id %v: %w", id.String(), err) + } + + return secret, nil +} + +func (db *Database) CreateSecret(name string, secret string) (Secret, error) { + // TODO: we need to validate that we can convert the name to a valid environment variable + query := ` +INSERT INTO secrets (id, name, secret) +VALUES (uuid_generate_v4(), $1, $2) +RETURNING id, name, secret;` + + s := Secret{} + var idStr string + err := db.Conn.QueryRow(context.Background(), query, name, secret).Scan(&idStr, &s.Name, &s.Secret) + if err != nil { + return s, fmt.Errorf("Could not create secret: %w", err) + } + + s.Id, err = uuid.Parse(idStr) + if err != nil { + return s, fmt.Errorf("Could not parse UUID generated by DB: %w", err) + } + + return s, nil +} + +func (db *Database) AssignSecretToPipeline(pipelineId uuid.UUID, secretId uuid.UUID) error { + query := ` +INSERT INTO pipeline_secret_mappings (pipeline, secret) +VALUES ($1, $2);` + + _, err := db.Conn.Exec(context.Background(), query, pipelineId, secretId) + + return err +} + +func (db *Database) RemoveSecretFromPipeline(pipelineId uuid.UUID, secretId uuid.UUID) error { + // TODO: implement this + return fmt.Errorf("Not implemented") +} + +func (db *Database) GetSecretsForPipeline(pipelineId uuid.UUID) ([]Secret, error) { + query := ` +SELECT + secrets.id, secrets.name, secrets.secret +FROM + secrets INNER JOIN pipeline_secret_mappings + ON secrets.id = pipeline_secret_mappings.secret +WHERE + pipeline_secret_mappings.pipeline=$1 +;` + + secrets := make([]Secret, 0) + + rows, err := db.Conn.Query(context.Background(), query, pipelineId) + if err != nil { + return secrets, fmt.Errorf("Could not get secrets for pipeline with id \"%v\": %w", pipelineId, err) + } + defer rows.Close() + + for rows.Next() { + var secret Secret + var idStr string + if err := rows.Scan( + &idStr, + &secret.Name, + &secret.Secret, + ); err != nil { + return secrets, err + } + + secret.Id, err = uuid.Parse(idStr) + if err != nil { + return secrets, err + } + secrets = append(secrets, secret) + } + + return secrets, nil +} diff --git a/database/types.go b/database/types.go index 2c4b67f..1efe900 100644 --- a/database/types.go +++ b/database/types.go @@ -6,27 +6,38 @@ import ( "github.com/google/uuid" ) -type CredentialType string +type CloneCredentialType string const ( - USER_PASS CredentialType = "USER_PASS" - SSH_KEY CredentialType = "SSH_KEY" + USER_PASS CloneCredentialType = "USER_PASS" + SSH_KEY CloneCredentialType = "SSH_KEY" ) -type Credential struct { +type CloneCredential struct { Id uuid.UUID Name string - Type CredentialType + Type CloneCredentialType Username string Secret string } type Pipeline struct { - Id uuid.UUID - Name string - Url string - PollInterval int - Credential *uuid.UUID + Id uuid.UUID + Name string + Url string + PollInterval int + CloneCredential *uuid.UUID +} + +type Secret struct { + Id uuid.UUID + Name string + Secret string +} + +type PipelineSecretMapping struct { + Pipeline uuid.UUID + Secret uuid.UUID } type WebhookSender string diff --git a/pipeline_executor/pipeline_executor.go b/pipeline_executor/pipeline_executor.go index c0e72dc..272afaf 100644 --- a/pipeline_executor/pipeline_executor.go +++ b/pipeline_executor/pipeline_executor.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "strings" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -54,8 +55,8 @@ func ExecutePipeline(pe PipelineExecution, db database.Database, pipelineConf co var auth transport.AuthMethod - if pe.Pipeline.Credential != nil { - credential, err := db.GetCredentialById(*pe.Pipeline.Credential) + if pe.Pipeline.CloneCredential != nil { + credential, err := db.GetCloneCredentialById(*pe.Pipeline.CloneCredential) if err != nil { log.Errorf("could not get credenital from db: %v", err) return @@ -154,6 +155,27 @@ func ExecutePipeline(pe PipelineExecution, db database.Database, pipelineConf co ) } + env := make([]string, 0) + + // set cursorius environment variables + env = append(env, []string{ + fmt.Sprintf("RUNID=%v", pe.Run.Id), + "CURSORIUS_SRC_DIR=/cursorius/src", + fmt.Sprintf("CUROSRIUS_SERVER_URL=%v", pipelineConf.AccessURL), + }...) + + // load secrets into environment + secrets, err := db.GetSecretsForPipeline(pe.Pipeline.Id) + if err != nil { + log.Errorf("Could not get secrets for pipeline", err) + return + } + + for _, secret := range secrets { + // TODO: this doesn't validate either of these strings + env = append(env, fmt.Sprintf("%v=%v", strings.ToUpper(secret.Name), secret.Secret)) + } + resp, err := cli.ContainerCreate(ctx, &container.Config{ Image: imageName,