cursorius/server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Add secrets support (#14)

Browse Source
This commit is contained in:
restitux
2023-02-14 20:18:41 -07:00
parent 6d2936393b
commit bfd05b6a8a
5 changed files with 395 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
admin_api/admin_api.go
+178 -29
View File
@@ -14,15 +14,52 @@ import (
var log = logging.MustGetLogger("cursorius-server")
func createSchema(db database.Database) (graphql.Schema, error) {
credentialType := graphql.NewObject(graphql.ObjectConfig{
Name: "Credential",
secretType := graphql.NewObject(graphql.ObjectConfig{
Name: "Secret",
Description: "A secret available for use inside of a pipeline.",
Fields: graphql.Fields{
"id": &graphql.Field{
Type: graphql.NewNonNull(graphql.String),
Description: "The id of the secret.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if secret, ok := p.Source.(database.Secret); ok {
return secret.Id, nil
}
return nil, nil
},
},
"name": &graphql.Field{
Type: graphql.NewNonNull(graphql.String),
Description: "The name of the secret.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if secret, ok := p.Source.(database.Secret); ok {
return secret.Name, nil
}
return nil, nil
},
},
"secret": &graphql.Field{
Type: graphql.NewNonNull(graphql.String),
Description: "The secret.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if secret, ok := p.Source.(database.Secret); ok {
return secret.Secret, nil
}
return nil, nil
},
},
},
})
cloneCredentialType := graphql.NewObject(graphql.ObjectConfig{
Name: "CloneCredential",
Description: "A credential for authenticating with the pipeline source host.",
Fields: graphql.Fields{
"id": &graphql.Field{
Type: graphql.NewNonNull(graphql.String),
Description: "The id of the credential.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if credential, ok := p.Source.(database.Credential); ok {
if credential, ok := p.Source.(database.CloneCredential); ok {
return credential.Id, nil
}
return nil, nil
@@ -32,7 +69,7 @@ func createSchema(db database.Database) (graphql.Schema, error) {
Type: graphql.NewNonNull(graphql.String),
Description: "The name of the credential.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if credential, ok := p.Source.(database.Credential); ok {
if credential, ok := p.Source.(database.CloneCredential); ok {
return credential.Name, nil
}
return nil, nil
@@ -42,7 +79,7 @@ func createSchema(db database.Database) (graphql.Schema, error) {
Type: graphql.NewNonNull(graphql.String),
Description: "The credential type.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if credential, ok := p.Source.(database.Credential); ok {
if credential, ok := p.Source.(database.CloneCredential); ok {
return credential.Type, nil
}
return nil, nil
@@ -52,7 +89,7 @@ func createSchema(db database.Database) (graphql.Schema, error) {
Type: graphql.NewNonNull(graphql.String),
Description: "The username to user with the credential.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if credential, ok := p.Source.(database.Credential); ok {
if credential, ok := p.Source.(database.CloneCredential); ok {
return credential.Username, nil
}
return nil, nil
@@ -62,7 +99,7 @@ func createSchema(db database.Database) (graphql.Schema, error) {
Type: graphql.NewNonNull(graphql.String),
Description: "The secret for the credential.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if credential, ok := p.Source.(database.Credential); ok {
if credential, ok := p.Source.(database.CloneCredential); ok {
return credential.Secret, nil
}
return nil, nil
@@ -210,16 +247,28 @@ func createSchema(db database.Database) (graphql.Schema, error) {
return nil, nil
},
},
"credentialId": &graphql.Field{
Type: graphql.String,
"cloneCredential": &graphql.Field{
Type: cloneCredentialType,
Description: "The configured credential for cloning the pipeline source.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if pipeline, ok := p.Source.(database.Pipeline); ok {
return pipeline.Credential, nil
if pipeline.CloneCredential != nil {
return db.GetCloneCredentialById(*pipeline.CloneCredential)
}
}
return nil, nil
},
},
"secrets": &graphql.Field{
Type: graphql.NewNonNull(graphql.NewList(graphql.NewNonNull(secretType))),
Description: "The list of secrets for the pipeline.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if pipeline, ok := p.Source.(database.Pipeline); ok {
return db.GetSecretsForPipeline(pipeline.Id)
}
return []database.Secret{}, nil
},
},
"webhooks": &graphql.Field{
Type: graphql.NewNonNull(graphql.NewList(graphql.NewNonNull(webhookType))),
Description: "The list of webhooks for the pipeline.",
@@ -269,8 +318,8 @@ func createSchema(db database.Database) (graphql.Schema, error) {
return db.GetPipelines()
},
},
"Credential": &graphql.Field{
Type: credentialType,
"CloneCredential": &graphql.Field{
Type: cloneCredentialType,
Args: graphql.FieldConfigArgument{
"id": &graphql.ArgumentConfig{
Type: graphql.NewNonNull(graphql.String),
@@ -282,16 +331,23 @@ func createSchema(db database.Database) (graphql.Schema, error) {
if err != nil {
return nil, err
}
return db.GetCredentialById(id)
return db.GetCloneCredentialById(id)
},
},
"Credentials": &graphql.Field{
Type: graphql.NewNonNull(graphql.NewList(credentialType)),
"CloneCredentials": &graphql.Field{
Type: graphql.NewNonNull(graphql.NewList(cloneCredentialType)),
Args: graphql.FieldConfigArgument{},
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
return db.GetCredentials()
},
},
"Secrets": &graphql.Field{
Type: graphql.NewNonNull(graphql.NewList(secretType)),
Args: graphql.FieldConfigArgument{},
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
return db.GetSecrets()
},
},
},
})
@@ -311,7 +367,7 @@ func createSchema(db database.Database) (graphql.Schema, error) {
"pollInterval": &graphql.ArgumentConfig{
Type: graphql.Int,
},
"credentialId": &graphql.ArgumentConfig{
"cloneCredentialId": &graphql.ArgumentConfig{
Type: graphql.String,
},
},
@@ -324,7 +380,7 @@ func createSchema(db database.Database) (graphql.Schema, error) {
}
var credential *uuid.UUID
if credentialVal, ok := params.Args["credentialId"]; ok {
if credentialVal, ok := params.Args["cloneCredentialId"]; ok {
id, err := uuid.Parse(credentialVal.(string))
if err != nil {
return nil, err
@@ -375,9 +431,9 @@ func createSchema(db database.Database) (graphql.Schema, error) {
return webhook, nil
},
},
"createCredential": &graphql.Field{
Type: credentialType,
Description: "Create a new credential",
"createCloneCredential": &graphql.Field{
Type: cloneCredentialType,
Description: "Create a new CloneCredential",
Args: graphql.FieldConfigArgument{
"name": &graphql.ArgumentConfig{
Type: graphql.NewNonNull(graphql.String),
@@ -396,7 +452,7 @@ func createSchema(db database.Database) (graphql.Schema, error) {
credential, err := db.CreateCredential(
params.Args["name"].(string),
database.CredentialType(params.Args["type"].(string)),
database.CloneCredentialType(params.Args["type"].(string)),
params.Args["username"].(string),
params.Args["secret"].(string),
)
@@ -406,11 +462,34 @@ func createSchema(db database.Database) (graphql.Schema, error) {
return credential, nil
},
},
"setPipelineCredential": &graphql.Field{
Type: pipelineType,
Description: "Add an credential to a pipeline",
"createSecret": &graphql.Field{
Type: secretType,
Description: "Create a new secret",
Args: graphql.FieldConfigArgument{
"credentialId": &graphql.ArgumentConfig{
"name": &graphql.ArgumentConfig{
Type: graphql.NewNonNull(graphql.String),
},
"secret": &graphql.ArgumentConfig{
Type: graphql.NewNonNull(graphql.String),
},
},
Resolve: func(params graphql.ResolveParams) (interface{}, error) {
secret, err := db.CreateSecret(
params.Args["name"].(string),
params.Args["secret"].(string),
)
if err != nil {
return nil, err
}
return secret, nil
},
},
"setPipelineCloneCredential": &graphql.Field{
Type: pipelineType,
Description: "Set the CloneCredential used by a pipeline to clone the source repo",
Args: graphql.FieldConfigArgument{
"cloneCredentialId": &graphql.ArgumentConfig{
Type: graphql.String,
},
"pipelineId": &graphql.ArgumentConfig{
@@ -424,19 +503,19 @@ func createSchema(db database.Database) (graphql.Schema, error) {
return nil, err
}
if credentialIdVal, ok := params.Args["credentialId"]; ok {
credentialId, err := uuid.Parse(credentialIdVal.(string))
if cloneCredentialIdVal, ok := params.Args["cloneCredentialId"]; ok {
cloneCredentialId, err := uuid.Parse(cloneCredentialIdVal.(string))
if err != nil {
return nil, err
}
pipeline, err := db.SetPipelineCredential(pipelineId, &credentialId)
pipeline, err := db.SetPipelineCloneCredential(pipelineId, &cloneCredentialId)
if err != nil {
return nil, err
}
return pipeline, nil
} else {
pipeline, err := db.SetPipelineCredential(pipelineId, nil)
pipeline, err := db.SetPipelineCloneCredential(pipelineId, nil)
if err != nil {
return nil, err
}
@@ -445,6 +524,76 @@ func createSchema(db database.Database) (graphql.Schema, error) {
},
},
"addSecretToPipeline": &graphql.Field{
Type: pipelineType,
Description: "Allow a secret to be accessed by a pipeline.",
Args: graphql.FieldConfigArgument{
"secretId": &graphql.ArgumentConfig{
Type: graphql.String,
},
"pipelineId": &graphql.ArgumentConfig{
Type: graphql.NewNonNull(graphql.String),
},
},
Resolve: func(params graphql.ResolveParams) (interface{}, error) {
secretId, err := uuid.Parse(params.Args["secretId"].(string))
if err != nil {
return nil, err
}
pipelineId, err := uuid.Parse(params.Args["pipelineId"].(string))
if err != nil {
return nil, err
}
err = db.AssignSecretToPipeline(pipelineId, secretId)
if err != nil {
return nil, err
}
pipeline, err := db.GetPipelineById(pipelineId)
if err != nil {
return nil, err
}
return pipeline, nil
},
},
"removeSecretFromPipeline": &graphql.Field{
Type: pipelineType,
Description: "Remove a pipeline's access to a secret.",
Args: graphql.FieldConfigArgument{
"secretId": &graphql.ArgumentConfig{
Type: graphql.String,
},
"pipelineId": &graphql.ArgumentConfig{
Type: graphql.NewNonNull(graphql.String),
},
},
Resolve: func(params graphql.ResolveParams) (interface{}, error) {
secretId, err := uuid.Parse(params.Args["secretId"].(string))
if err != nil {
return nil, err
}
pipelineId, err := uuid.Parse(params.Args["pipelineId"].(string))
if err != nil {
return nil, err
}
err = db.RemoveSecretFromPipeline(pipelineId, secretId)
if err != nil {
return nil, err
}
pipeline, err := db.GetPipelineById(pipelineId)
if err != nil {
return nil, err
}
return pipeline, nil
},
},
},
})
database/db.go
+28 -9
View File
@@ -95,7 +95,7 @@ CREATE TABLE version (
);
CREATE TABLE credentials (
CREATE TABLE clone_credentials (
id UUID PRIMARY KEY,
name TEXT NOT NULL,
type TEXT NOT NULL,
@@ -104,15 +104,34 @@ CREATE TABLE credentials (
);
CREATE TABLE pipelines (
id UUID PRIMARY KEY,
name TEXT NOT NULL,
url TEXT NOT NULL,
poll_interval INTEGER,
credential UUID DEFAULT NULL,
id UUID PRIMARY KEY,
name TEXT NOT NULL,
url TEXT NOT NULL,
poll_interval INTEGER,
clone_credential UUID DEFAULT NULL,
CONSTRAINT fk_credential
FOREIGN KEY(credential)
REFERENCES credentials(id)
CONSTRAINT fk_clone_credential
FOREIGN KEY(clone_credential)
REFERENCES clone_credentials(id)
);
CREATE TABLE secrets (
id UUID PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
secret TEXT NOT NULL
);
CREATE TABLE pipeline_secret_mappings (
pipeline UUID NOT NULL,
secret UUID NOT NULL,
CONSTRAINT fk_pipeline
FOREIGN KEY(pipeline)
REFERENCES pipelines(id),
CONSTRAINT fk_secret
FOREIGN KEY(secret)
REFERENCES secrets(id)
);
CREATE TABLE webhooks (
database/func.go
+144 -19
View File
@@ -10,7 +10,7 @@ import (
func (db *Database) GetPipelines() ([]Pipeline, error) {
query := `
SELECT id, name, url, poll_interval, credential
SELECT id, name, url, poll_interval, clone_credential
FROM pipelines;`
pipelines := make([]Pipeline, 0)
@@ -24,7 +24,7 @@ FROM pipelines;`
for rows.Next() {
var pipeline Pipeline
var idStr string
if err := rows.Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.Credential); err != nil {
if err := rows.Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.CloneCredential); err != nil {
return pipelines, err
}
@@ -58,7 +58,7 @@ WHERE id=$1;`
func (db *Database) CreatePipeline(name string, url string, pollInterval int, credential *uuid.UUID) (Pipeline, error) {
query := `
INSERT INTO pipelines (id, name, url, poll_interval, credential)
INSERT INTO pipelines (id, name, url, poll_interval, clone_credential)
VALUES (uuid_generate_v4(), $1, $2, $3, $4)
RETURNING id, name, url, poll_interval;`
@@ -77,12 +77,12 @@ RETURNING id, name, url, poll_interval;`
return pipeline, nil
}
func (db *Database) SetPipelineCredential(pipelineId uuid.UUID, credentialId *uuid.UUID) (Pipeline, error) {
func (db *Database) SetPipelineCloneCredential(pipelineId uuid.UUID, credentialId *uuid.UUID) (Pipeline, error) {
query := `
UPDATE pipelines
SET credential=$1
SET clone_credential=$1
WHERE id=$2
RETURNING name, url, poll_interval, credential;`
RETURNING name, url, poll_interval, clone_credential;`
pipeline := Pipeline{
Id: pipelineId,
@@ -90,7 +90,7 @@ RETURNING name, url, poll_interval, credential;`
err := db.Conn.QueryRow(context.Background(),
query, credentialId, pipelineId).Scan(
&pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.Credential,
&pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.CloneCredential,
)
if err != nil {
return pipeline, fmt.Errorf("Could not add credential to pipeline: %w", err)
@@ -104,7 +104,7 @@ func (db *Database) RemovePipelineCredential(pipelineId uuid.UUID) (Pipeline, er
UPDATE pipelines
SET credential=null
WHERE id=$1
RETURNING name, url, poll_interval, credential;`
RETURNING name, url, poll_interval, clone_credential;`
pipeline := Pipeline{
Id: pipelineId,
@@ -112,7 +112,7 @@ RETURNING name, url, poll_interval, credential;`
err := db.Conn.QueryRow(context.Background(),
query, pipelineId).Scan(
&pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.Credential,
&pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.CloneCredential,
)
if err != nil {
return pipeline, fmt.Errorf("Could not add credential to pipeline: %w", err)
@@ -194,13 +194,13 @@ RETURNING id, server_type, secret, pipeline;`
return webhook, nil
}
func (db *Database) CreateCredential(name string, credentialtype CredentialType, username string, secret string) (Credential, error) {
func (db *Database) CreateCredential(name string, credentialtype CloneCredentialType, username string, secret string) (CloneCredential, error) {
query := `
INSERT INTO credentials (id, name, type, username, secret)
INSERT INTO clone_credentials (id, name, type, username, secret)
VALUES(uuid_generate_v4(), $1, $2, $3, $4)
RETURNING id, name, type, username, secret;`
credential := Credential{}
credential := CloneCredential{}
var idStr string
err := db.Conn.QueryRow(
context.Background(),
@@ -224,15 +224,15 @@ RETURNING id, name, type, username, secret;`
return credential, nil
}
func (db *Database) GetCredentialById(id uuid.UUID) (Credential, error) {
func (db *Database) GetCloneCredentialById(id uuid.UUID) (CloneCredential, error) {
query := `
SELECT name, type, username, secret
FROM credentials
FROM clone_credentials
WHERE id=$1;`
log.Debugf("requested credential with id %v", id)
credential := Credential{
credential := CloneCredential{
Id: id,
}
@@ -244,12 +244,12 @@ WHERE id=$1;`
return credential, nil
}
func (db *Database) GetCredentials() ([]Credential, error) {
func (db *Database) GetCredentials() ([]CloneCredential, error) {
query := `
SELECT id, name, type, username, secret
FROM credentials;`
FROM clone_credentials;`
credentials := make([]Credential, 0)
credentials := make([]CloneCredential, 0)
rows, err := db.Conn.Query(context.Background(), query)
if err != nil {
@@ -258,7 +258,7 @@ FROM credentials;`
defer rows.Close()
for rows.Next() {
var credential Credential
var credential CloneCredential
var idStr string
if err := rows.Scan(&idStr, &credential.Name, &credential.Type, &credential.Username, &credential.Secret); err != nil {
return credentials, err
@@ -395,3 +395,128 @@ func (db *Database) UpdatePipelineRefs(pipelineId uuid.UUID, refsMap map[string]
log.Debugf("copyCount: %v", copyCount)
return nil
}
func (db *Database) GetSecrets() ([]Secret, error) {
query := `
SELECT id, name, secret
FROM secrets;`
secrets := make([]Secret, 0)
rows, err := db.Conn.Query(context.Background(), query)
if err != nil {
return secrets, fmt.Errorf("Could not query database for secrets: %w", err)
}
defer rows.Close()
for rows.Next() {
var secret Secret
var idStr string
if err := rows.Scan(&idStr, &secret.Name, &secret.Secret); err != nil {
return secrets, err
}
secret.Id, err = uuid.Parse(idStr)
if err != nil {
return secrets, err
}
secrets = append(secrets, secret)
}
return secrets, nil
}
func (db *Database) GetSecretById(id uuid.UUID) (Secret, error) {
query := `
SELECT id, name, secret
FROM secrets
WHERE id=$1;`
secret := Secret{
Id: id,
}
err := db.Conn.QueryRow(context.Background(), query, id).Scan(&secret.Name, &secret.Secret)
if err != nil {
return secret, fmt.Errorf("Could not query database for secret with id %v: %w", id.String(), err)
}
return secret, nil
}
func (db *Database) CreateSecret(name string, secret string) (Secret, error) {
// TODO: we need to validate that we can convert the name to a valid environment variable
query := `
INSERT INTO secrets (id, name, secret)
VALUES (uuid_generate_v4(), $1, $2)
RETURNING id, name, secret;`
s := Secret{}
var idStr string
err := db.Conn.QueryRow(context.Background(), query, name, secret).Scan(&idStr, &s.Name, &s.Secret)
if err != nil {
return s, fmt.Errorf("Could not create secret: %w", err)
}
s.Id, err = uuid.Parse(idStr)
if err != nil {
return s, fmt.Errorf("Could not parse UUID generated by DB: %w", err)
}
return s, nil
}
func (db *Database) AssignSecretToPipeline(pipelineId uuid.UUID, secretId uuid.UUID) error {
query := `
INSERT INTO pipeline_secret_mappings (pipeline, secret)
VALUES ($1, $2);`
_, err := db.Conn.Exec(context.Background(), query, pipelineId, secretId)
return err
}
func (db *Database) RemoveSecretFromPipeline(pipelineId uuid.UUID, secretId uuid.UUID) error {
// TODO: implement this
return fmt.Errorf("Not implemented")
}
func (db *Database) GetSecretsForPipeline(pipelineId uuid.UUID) ([]Secret, error) {
query := `
SELECT
secrets.id, secrets.name, secrets.secret
FROM
secrets INNER JOIN pipeline_secret_mappings
ON secrets.id = pipeline_secret_mappings.secret
WHERE
pipeline_secret_mappings.pipeline=$1
;`
secrets := make([]Secret, 0)
rows, err := db.Conn.Query(context.Background(), query, pipelineId)
if err != nil {
return secrets, fmt.Errorf("Could not get secrets for pipeline with id \"%v\": %w", pipelineId, err)
}
defer rows.Close()
for rows.Next() {
var secret Secret
var idStr string
if err := rows.Scan(
&idStr,
&secret.Name,
&secret.Secret,
); err != nil {
return secrets, err
}
secret.Id, err = uuid.Parse(idStr)
if err != nil {
return secrets, err
}
secrets = append(secrets, secret)
}
return secrets, nil
}
database/types.go
+21 -10
View File
@@ -6,27 +6,38 @@ import (
"github.com/google/uuid"
)
type CredentialType string
type CloneCredentialType string
const (
USER_PASS CredentialType = "USER_PASS"
SSH_KEY CredentialType = "SSH_KEY"
USER_PASS CloneCredentialType = "USER_PASS"
SSH_KEY CloneCredentialType = "SSH_KEY"
)
type Credential struct {
type CloneCredential struct {
Id uuid.UUID
Name string
Type CredentialType
Type CloneCredentialType
Username string
Secret string
}
type Pipeline struct {
Id uuid.UUID
Name string
Url string
PollInterval int
Credential *uuid.UUID
Id uuid.UUID
Name string
Url string
PollInterval int
CloneCredential *uuid.UUID
}
type Secret struct {
Id uuid.UUID
Name string
Secret string
}
type PipelineSecretMapping struct {
Pipeline uuid.UUID
Secret uuid.UUID
}
type WebhookSender string
pipeline_executor/pipeline_executor.go
+24 -2
View File
@@ -7,6 +7,7 @@ import (
"io"
"os"
"path/filepath"
"strings"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
@@ -54,8 +55,8 @@ func ExecutePipeline(pe PipelineExecution, db database.Database, pipelineConf co
var auth transport.AuthMethod
if pe.Pipeline.Credential != nil {
credential, err := db.GetCredentialById(*pe.Pipeline.Credential)
if pe.Pipeline.CloneCredential != nil {
credential, err := db.GetCloneCredentialById(*pe.Pipeline.CloneCredential)
if err != nil {
log.Errorf("could not get credenital from db: %v", err)
return
@@ -154,6 +155,27 @@ func ExecutePipeline(pe PipelineExecution, db database.Database, pipelineConf co
)
}
env := make([]string, 0)
// set cursorius environment variables
env = append(env, []string{
fmt.Sprintf("RUNID=%v", pe.Run.Id),
"CURSORIUS_SRC_DIR=/cursorius/src",
fmt.Sprintf("CUROSRIUS_SERVER_URL=%v", pipelineConf.AccessURL),
}...)
// load secrets into environment
secrets, err := db.GetSecretsForPipeline(pe.Pipeline.Id)
if err != nil {
log.Errorf("Could not get secrets for pipeline", err)
return
}
for _, secret := range secrets {
// TODO: this doesn't validate either of these strings
env = append(env, fmt.Sprintf("%v=%v", strings.ToUpper(secret.Name), secret.Secret))
}
resp, err := cli.ContainerCreate(ctx,
&container.Config{
Image: imageName,