cursorius/server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Add support for access repos with credentials

Browse Source
This commit is contained in:
restitux
2023-02-07 21:34:32 -07:00
parent d870335d25
commit 7a665aa348
5 changed files with 401 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
admin_api/admin_api.go
+176 -3
View File
@@ -14,6 +14,63 @@ import (
var log = logging.MustGetLogger("cursorius-server")
func createSchema(db database.Database) (graphql.Schema, error) {
credentialType := graphql.NewObject(graphql.ObjectConfig{
Name: "Credential",
Description: "A credential for authenticating with the pipeline source host.",
Fields: graphql.Fields{
"id": &graphql.Field{
Type: graphql.NewNonNull(graphql.String),
Description: "The id of the credential.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if credential, ok := p.Source.(database.Credential); ok {
return credential.Id, nil
}
return nil, nil
},
},
"name": &graphql.Field{
Type: graphql.NewNonNull(graphql.String),
Description: "The name of the credential.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if credential, ok := p.Source.(database.Credential); ok {
return credential.Name, nil
}
return nil, nil
},
},
"type": &graphql.Field{
Type: graphql.NewNonNull(graphql.String),
Description: "The credential type.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if credential, ok := p.Source.(database.Credential); ok {
return credential.Type, nil
}
return nil, nil
},
},
"username": &graphql.Field{
Type: graphql.NewNonNull(graphql.String),
Description: "The username to user with the credential.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if credential, ok := p.Source.(database.Credential); ok {
return credential.Username, nil
}
return nil, nil
},
},
"secret": &graphql.Field{
Type: graphql.NewNonNull(graphql.String),
Description: "The secret for the credential.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if credential, ok := p.Source.(database.Credential); ok {
return credential.Secret, nil
}
return nil, nil
},
},
},
})
webhookType := graphql.NewObject(graphql.ObjectConfig{
Name: "Webhook",
Description: "A webhook for triggering pipelines",
@@ -153,6 +210,16 @@ func createSchema(db database.Database) (graphql.Schema, error) {
return nil, nil
},
},
"credentialId": &graphql.Field{
Type: graphql.String,
Description: "The configured credential for cloning the pipeline source.",
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
if pipeline, ok := p.Source.(database.Pipeline); ok {
return pipeline.Credential, nil
}
return nil, nil
},
},
"webhooks": &graphql.Field{
Type: graphql.NewNonNull(graphql.NewList(graphql.NewNonNull(webhookType))),
Description: "The list of webhooks for the pipeline.",
@@ -202,6 +269,29 @@ func createSchema(db database.Database) (graphql.Schema, error) {
return db.GetPipelines()
},
},
"Credential": &graphql.Field{
Type: credentialType,
Args: graphql.FieldConfigArgument{
"id": &graphql.ArgumentConfig{
Type: graphql.NewNonNull(graphql.String),
Description: "The id of the requested credential.",
},
},
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
id, err := uuid.Parse(p.Args["id"].(string))
if err != nil {
return nil, err
}
return db.GetCredentialById(id)
},
},
"Credentials": &graphql.Field{
Type: graphql.NewNonNull(graphql.NewList(credentialType)),
Args: graphql.FieldConfigArgument{},
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
return db.GetCredentials()
},
},
},
})
@@ -221,6 +311,9 @@ func createSchema(db database.Database) (graphql.Schema, error) {
"pollInterval": &graphql.ArgumentConfig{
Type: graphql.Int,
},
"credentialId": &graphql.ArgumentConfig{
Type: graphql.String,
},
},
Resolve: func(params graphql.ResolveParams) (interface{}, error) {
var interval int
@@ -229,18 +322,28 @@ func createSchema(db database.Database) (graphql.Schema, error) {
} else {
interval = 0
}
var credential *uuid.UUID
if credentialVal, ok := params.Args["credentialId"]; ok {
id, err := uuid.Parse(credentialVal.(string))
if err != nil {
return nil, err
}
credential = &id
} else {
credential = nil
}
pipeline, err := db.CreatePipeline(
params.Args["name"].(string),
params.Args["url"].(string),
interval,
credential,
)
if err != nil {
return nil, err
}
log.Infof("Created pipeline with id: %v, name: %v, url: %v, and poll interval: %v",
pipeline.Id, pipeline.Name, pipeline.Url, pipeline.PollInterval)
return pipeline, nil
},
},
@@ -272,6 +375,76 @@ func createSchema(db database.Database) (graphql.Schema, error) {
return webhook, nil
},
},
"createCredential": &graphql.Field{
Type: credentialType,
Description: "Create a new credential",
Args: graphql.FieldConfigArgument{
"name": &graphql.ArgumentConfig{
Type: graphql.NewNonNull(graphql.String),
},
"type": &graphql.ArgumentConfig{
Type: graphql.NewNonNull(graphql.String),
},
"username": &graphql.ArgumentConfig{
Type: graphql.NewNonNull(graphql.String),
},
"secret": &graphql.ArgumentConfig{
Type: graphql.NewNonNull(graphql.String),
},
},
Resolve: func(params graphql.ResolveParams) (interface{}, error) {
credential, err := db.CreateCredential(
params.Args["name"].(string),
database.CredentialType(params.Args["type"].(string)),
params.Args["username"].(string),
params.Args["secret"].(string),
)
if err != nil {
return nil, err
}
return credential, nil
},
},
"setPipelineCredential": &graphql.Field{
Type: pipelineType,
Description: "Add an credential to a pipeline",
Args: graphql.FieldConfigArgument{
"credentialId": &graphql.ArgumentConfig{
Type: graphql.String,
},
"pipelineId": &graphql.ArgumentConfig{
Type: graphql.NewNonNull(graphql.String),
},
},
Resolve: func(params graphql.ResolveParams) (interface{}, error) {
pipelineId, err := uuid.Parse(params.Args["pipelineId"].(string))
if err != nil {
return nil, err
}
if credentialIdVal, ok := params.Args["credentialId"]; ok {
credentialId, err := uuid.Parse(credentialIdVal.(string))
if err != nil {
return nil, err
}
pipeline, err := db.SetPipelineCredential(pipelineId, &credentialId)
if err != nil {
return nil, err
}
return pipeline, nil
} else {
pipeline, err := db.SetPipelineCredential(pipelineId, nil)
if err != nil {
return nil, err
}
return pipeline, nil
}
},
},
},
})
database/db.go
+27 -10
View File
@@ -40,10 +40,15 @@ func LaunchDB(conf config.DBConfig) (Database, error) {
db := Database{}
var err error
log.Infof("Connecting to database with URL \"%v\"", dbURLNoPasswd)
db.Conn, err = pgxpool.New(context.Background(), dbURL)
if err != nil {
return db, fmt.Errorf("could not create database pool: %w", err)
}
// sleep until we can sucessfully acquire a connection
for i := 0; i < 10; i++ {
// TODO: retry logic is broken with pgxpool
log.Infof("Connecting to database with URL \"%v\" (attempt %v)", dbURLNoPasswd, i)
db.Conn, err = pgxpool.New(context.Background(), dbURL)
_, err = db.Conn.Acquire(context.Background())
if err == nil {
break
}
@@ -90,11 +95,24 @@ CREATE TABLE version (
);
CREATE TABLE credentials (
id UUID PRIMARY KEY,
name TEXT NOT NULL,
type TEXT NOT NULL,
username TEXT NOT NULL,
secret TEXT NOT NULL
);
CREATE TABLE pipelines (
id UUID PRIMARY KEY,
name TEXT NOT NULL,
url TEXT NOT NULL,
poll_interval INTEGER
poll_interval INTEGER,
credential UUID DEFAULT NULL,
CONSTRAINT fk_credential
FOREIGN KEY(credential)
REFERENCES credentials(id)
);
CREATE TABLE webhooks (
@@ -108,12 +126,6 @@ CREATE TABLE webhooks (
REFERENCES pipelines(id)
);
CREATE TABLE runners (
id UUID PRIMARY KEY,
name TEXT,
secret TEXT
);
CREATE TABLE runs (
id UUID PRIMARY KEY,
pipeline UUID,
@@ -142,6 +154,11 @@ CREATE TABLE command_executions (
REFERENCES runs(id)
);
CREATE TABLE runners (
id UUID PRIMARY KEY,
name TEXT,
secret TEXT
);
`
_, err := conn.Exec(context.Background(), createTablesQuery)
database/func.go
+132 -9
View File
@@ -9,7 +9,7 @@ import (
func (db *Database) GetPipelines() ([]Pipeline, error) {
query := `
SELECT id, name, url, poll_interval
SELECT id, name, url, poll_interval, credential
FROM pipelines;`
pipelines := make([]Pipeline, 0)
@@ -23,7 +23,7 @@ FROM pipelines;`
for rows.Next() {
var pipeline Pipeline
var idStr string
if err := rows.Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval); err != nil {
if err := rows.Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.Credential); err != nil {
return pipelines, err
}
@@ -55,29 +55,71 @@ WHERE id=$1;`
return pipeline, nil
}
func (db *Database) CreatePipeline(name string, url string, pollInterval int) (Pipeline, error) {
func (db *Database) CreatePipeline(name string, url string, pollInterval int, credential *uuid.UUID) (Pipeline, error) {
query := `
INSERT INTO pipelines (id, name, url, poll_interval)
VALUES (uuid_generate_v4(), $1, $2, $3)
INSERT INTO pipelines (id, name, url, poll_interval, credential)
VALUES (uuid_generate_v4(), $1, $2, $3, $4)
RETURNING id, name, url, poll_interval;`
pipeline := Pipeline{}
var idStr string
err := db.Conn.QueryRow(context.Background(), query, name, url, pollInterval).Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval)
err := db.Conn.QueryRow(context.Background(), query, name, url, pollInterval, credential).Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval)
if err != nil {
return pipeline, fmt.Errorf("Could not create pipeline: %w", err)
}
id, err := uuid.Parse(idStr)
pipeline.Id, err = uuid.Parse(idStr)
if err != nil {
return pipeline, fmt.Errorf("Could not parse UUID generated by DB: %w", err)
}
pipeline.Id = id
return pipeline, nil
}
func (db *Database) SetPipelineCredential(pipelineId uuid.UUID, credentialId *uuid.UUID) (Pipeline, error) {
query := `
UPDATE pipelines
SET credential=$1
WHERE id=$2
RETURNING name, url, poll_interval, credential;`
pipeline := Pipeline{
Id: pipelineId,
}
err := db.Conn.QueryRow(context.Background(),
query, credentialId, pipelineId).Scan(
&pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.Credential,
)
if err != nil {
return pipeline, fmt.Errorf("Could not add credential to pipeline: %w", err)
}
return pipeline, err
}
func (db *Database) RemovePipelineCredential(pipelineId uuid.UUID) (Pipeline, error) {
query := `
UPDATE pipelines
SET credential=null
WHERE id=$1
RETURNING name, url, poll_interval, credential;`
pipeline := Pipeline{
Id: pipelineId,
}
err := db.Conn.QueryRow(context.Background(),
query, pipelineId).Scan(
&pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.Credential,
)
if err != nil {
return pipeline, fmt.Errorf("Could not add credential to pipeline: %w", err)
}
return pipeline, err
}
func (db *Database) GetWebhooksForPipeline(id uuid.UUID) ([]Webhook, error) {
query := `
SELECT id, server_type, secret
@@ -151,6 +193,86 @@ RETURNING id, server_type, secret, pipeline;`
return webhook, nil
}
func (db *Database) CreateCredential(name string, credentialtype CredentialType, username string, secret string) (Credential, error) {
query := `
INSERT INTO credentials (id, name, type, username, secret)
VALUES(uuid_generate_v4(), $1, $2, $3, $4)
RETURNING id, name, type, username, secret;`
credential := Credential{}
var idStr string
err := db.Conn.QueryRow(
context.Background(),
query,
name,
string(credentialtype),
username,
secret,
).Scan(&idStr, &credential.Name, &credential.Type, &credential.Username, &credential.Secret)
if err != nil {
return credential, err
}
id, err := uuid.Parse(idStr)
if err != nil {
return credential, err
}
credential.Id = id
return credential, nil
}
func (db *Database) GetCredentialById(id uuid.UUID) (Credential, error) {
query := `
SELECT name, type, username, secret
FROM credentials
WHERE id=$1;`
log.Debugf("requested credential with id %v", id)
credential := Credential{
Id: id,
}
err := db.Conn.QueryRow(context.Background(), query, id).Scan(&credential.Name, &credential.Type, &credential.Username, &credential.Secret)
if err != nil {
return credential, fmt.Errorf("Could not query database for credential with id %v: %w", id.String(), err)
}
return credential, nil
}
func (db *Database) GetCredentials() ([]Credential, error) {
query := `
SELECT id, name, type, username, secret
FROM credentials;`
credentials := make([]Credential, 0)
rows, err := db.Conn.Query(context.Background(), query)
if err != nil {
return credentials, fmt.Errorf("Could not query database for credentials: %w", err)
}
defer rows.Close()
for rows.Next() {
var credential Credential
var idStr string
if err := rows.Scan(&idStr, &credential.Name, &credential.Type, &credential.Username, &credential.Secret); err != nil {
return credentials, err
}
credential.Id, err = uuid.Parse(idStr)
if err != nil {
return credentials, err
}
credentials = append(credentials, credential)
}
return credentials, nil
}
func (db *Database) CreateRun(pipelineId uuid.UUID) (Run, error) {
query := `
INSERT INTO runs (id, pipeline, in_progress)
@@ -173,6 +295,7 @@ RETURNING id, pipeline, in_progress;`
}
func (db *Database) UpdateRunResult(r Run) error {
// TODO: the id fiend is the query is broken
query := `
UPDATE runs
SET in_progress=$1, result=$2, stdout=$3, stderr=$4
database/types.go
+22 -6
View File
@@ -6,11 +6,27 @@ import (
"github.com/google/uuid"
)
type CredentialType string
const (
USER_PASS CredentialType = "USER_PASS"
SSH_KEY CredentialType = "SSH_KEY"
)
type Credential struct {
Id uuid.UUID
Name string
Type CredentialType
Username string
Secret string
}
type Pipeline struct {
Id uuid.UUID
Name string
Url string
PollInterval int
Credential *uuid.UUID
}
type WebhookSender string
@@ -26,12 +42,6 @@ type Webhook struct {
Pipeline uuid.UUID
}
type Runner struct {
Id uuid.UUID
Name string
Secret string
}
type Run struct {
Id uuid.UUID
Pipeline uuid.UUID
@@ -51,3 +61,9 @@ type CommandExecution struct {
StartTime time.Time
EndTime time.Time
}
type Runner struct {
Id uuid.UUID
Name string
Secret string
}
pipeline_executor/pipeline_executor.go
+44 -8
View File
@@ -6,17 +6,21 @@ import (
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"git.ohea.xyz/cursorius/server/config"
"git.ohea.xyz/cursorius/server/database"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/client"
"github.com/docker/docker/pkg/stdcopy"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/op/go-logging"
"git.ohea.xyz/cursorius/server/config"
"git.ohea.xyz/cursorius/server/database"
)
var log = logging.MustGetLogger("cursorius-server")
@@ -47,12 +51,44 @@ func ExecutePipeline(pe PipelineExecution, db database.Database, pipelineConf co
}
log.Infof("Cloning source from URL %v", pe.Pipeline.Url)
// TODO: should I use go-git here instead of shelling out to raw git?
cloneCmd := exec.Command("git", "clone", pe.Pipeline.Url, jobFolder)
output, err := cloneCmd.CombinedOutput()
var auth transport.AuthMethod
if pe.Pipeline.Credential != nil {
credential, err := db.GetCredentialById(*pe.Pipeline.Credential)
if err != nil {
log.Errorf("could not get credenital from db: %v", err)
return
}
switch credential.Type {
case "USER_PASS":
log.Debugf("job %v configured to use credential %v", pe.Pipeline.Name, credential.Name)
auth = transport.AuthMethod(&http.BasicAuth{
Username: credential.Username,
Password: credential.Secret,
})
case "SSH_KEY":
publicKeys, err := ssh.NewPublicKeys(credential.Username, []byte(credential.Secret), "")
if err != nil {
log.Errorf("could not parse credential %v", credential.Name)
return
}
auth = transport.AuthMethod(publicKeys)
default:
log.Errorf("unsupported credential type %v", credential.Type)
return
}
} else {
auth = nil
}
_, err = git.PlainClone(jobFolder, false, &git.CloneOptions{
URL: pe.Pipeline.Url,
Auth: auth,
})
if err != nil {
log.Debugf("%s", output)
log.Errorf("could not clone source: %w", err)
log.Errorf("could not clone repo: %v", err)
return
}