From ddf9fda09240803744c069ebd7678bb6605d5a09 Mon Sep 17 00:00:00 2001 From: restitux Date: Fri, 13 Jan 2023 23:25:41 -0700 Subject: [PATCH] Add basic postgres db config --- config/config.go | 16 ++++++ database/db.go | 113 ++++++++++++++++++++++++++++++++++++++ docker/docker-compose.yml | 13 ++++- go.mod | 4 ++ go.sum | 7 +++ main.go | 9 +++ 6 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 database/db.go diff --git a/config/config.go b/config/config.go index 73e25d9..e66e39c 100644 --- a/config/config.go +++ b/config/config.go @@ -28,6 +28,14 @@ type Runner struct { Secret string } +type DBConfig struct { + Address string + Port int + Username string + Password string + Name string +} + type MountType string const ( @@ -50,6 +58,7 @@ type PipelineConf struct { type Config struct { Address string Port int + DBConfig DBConfig PipelineConf PipelineConf Jobs map[string]Job Runners map[string]Runner @@ -63,6 +72,13 @@ func GetConfig() (config.Config[Config], error) { Config: Config{ Address: "127.0.0.1", Port: 45420, + DBConfig: DBConfig{ + Address: "DB_ADDRESS", + Port: 5432, + Username: "USERNAME", + Password: "PASSWORD", + Name: "cursorius", + }, PipelineConf: PipelineConf{ AccessURL: "cursorius-server:45420", DockerNetwork: &defaultNetworkName, diff --git a/database/db.go b/database/db.go new file mode 100644 index 0000000..242095d --- /dev/null +++ b/database/db.go @@ -0,0 +1,113 @@ +package database + +import ( + "time" + + "git.ohea.xyz/cursorius/server/config" + + "context" + "fmt" + + "github.com/jackc/pgx/v5" + "github.com/op/go-logging" +) + +var log = logging.MustGetLogger("cursorius-server") + +func LaunchDB(conf config.DBConfig) error { + + dbURL := fmt.Sprintf( + "postgres://%v:%v@%v:%v/%v", + conf.Username, + conf.Password, + conf.Address, + conf.Port, + conf.Name, + ) + + dbURLNoPasswd := fmt.Sprintf( + "postgres://%v:********@%v:%v/%v", + conf.Username, + conf.Address, + conf.Port, + conf.Name, + ) + + log.Infof("Connecting to database with URL \"%v\"", dbURLNoPasswd) + + var conn *pgx.Conn + var err error + for i := 0; i < 10; i++ { + conn, err = pgx.Connect(context.Background(), dbURL) + if err == nil { + defer conn.Close(context.Background()) + break + } + time.Sleep(2 * time.Second) + + } + if err != nil { + return fmt.Errorf("Could not open database: %w", err) + } + + versionTableExistsQuery := ` +SELECT EXISTS ( + SELECT FROM pg_tables + WHERE tablename = 'version' +);` + + var versionTableExists bool + err = conn.QueryRow(context.Background(), versionTableExistsQuery).Scan(&versionTableExists) + if err != nil { + return fmt.Errorf("Could not check if database was initalized: %w", err) + } + + if versionTableExists { + // TODO: migrations + } else { + err = initDB(conn) + if err != nil { + return fmt.Errorf("Could not initalize database: %w", err) + } + } + + return nil +} + +func initDB(conn *pgx.Conn) error { + createTablesQuery := ` +CREATE TABLE version ( + version INT NOT NULL +); + +CREATE TABLE pipelines ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + url TEXT NOT NULL, + poll_interval INTEGER +); + +CREATE TABLE webhooks ( + id SERIAL PRIMARY KEY, + type TEXT, + secret TEXT, + pipeline INTEGER, + + CONSTRAINT fk_pipeline + FOREIGN KEY(pipeline) + REFERENCES pipelines(id) +); + +CREATE TABLE runners ( + id SERIAL PRIMARY KEY, + name TEXT, + secret TEXT +);` + + _, err := conn.Exec(context.Background(), createTablesQuery) + if err != nil { + return err + } + + return nil +} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 5d216c4..e75b180 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -6,13 +6,24 @@ services: dockerfile: docker/Dockerfile.dev ports: - "0.0.0.0:45420:45420" + networks: + - cursorius volumes: - "..:/build/server" - "../server.toml:/root/.config/cursorius/server.toml" - "/var/run/docker.sock:/var/run/docker.sock" - "../_working/go:/go" - "../_working/jobs:/cursorius/jobs" + cursorius-db: + image: postgres:14 + environment: + - POSTGRES_USER=cursorius + - POSTGRES_PASSWORD=cursorius + - POSTGRES_DB=cursorius + networks: + - cursorius + volumes: + - "../_working/postgres:/var/lib/postgresql/data" networks: cursorius: - external: true diff --git a/go.mod b/go.mod index 5252fdb..fb96dc7 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/go-git/go-git/v5 v5.4.3-0.20220529141257-bc1f419cebcf github.com/go-playground/webhooks/v6 v6.0.1 github.com/google/uuid v1.3.0 + github.com/jackc/pgx/v5 v5.2.0 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 golang.org/x/net v0.2.0 google.golang.org/protobuf v1.28.1 @@ -26,12 +27,15 @@ require ( github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect + github.com/doug-martin/goqu/v9 v9.18.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/gobwas/ws v1.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/imdario/mergo v0.3.13 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect diff --git a/go.sum b/go.sum index 3a66849..3b61a96 100644 --- a/go.sum +++ b/go.sum @@ -111,6 +111,7 @@ github.com/Azure/go-autorest v12.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.2/go.mod h1:af5vUNlDNkCjOZeSGFgIJxDje9qdjsO6hshx0gTmZt4= @@ -360,6 +361,8 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/doug-martin/goqu/v9 v9.18.0 h1:/6bcuEtAe6nsSMVK/M+fOiXUNfyFF3yYtE07DBPFMYY= +github.com/doug-martin/goqu/v9 v9.18.0/go.mod h1:nf0Wc2/hV3gYK9LiyqIrzBEVGlI8qW3GuDCEobC4wBQ= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/duo-labs/webauthn v0.0.0-20220815211337-00c9fb5711f5/go.mod h1:Jcj7rFNlTknb18v9jpSA58BveX2LDhXqaoy+6YV1N9g= @@ -803,6 +806,7 @@ github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bY github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= @@ -814,6 +818,7 @@ github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= @@ -835,6 +840,8 @@ github.com/jackc/pgx/v4 v4.11.0/go.mod h1:i62xJgdrtVDsnL3U8ekyrQXEwGNTRoG7/8r+CI github.com/jackc/pgx/v4 v4.12.0/go.mod h1:fE547h6VulLPA3kySjfnSG/e2D861g/50JlVUa/ub60= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ= +github.com/jackc/pgx/v5 v5.2.0 h1:NdPpngX0Y6z6XDFKqmFQaE+bCtkqzvQIOt1wvBlAqs8= +github.com/jackc/pgx/v5 v5.2.0/go.mod h1:Ptn7zmohNsWEsdxRawMzk3gaKma2obW+NWTnKa0S4nk= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= diff --git a/main.go b/main.go index 6babd1b..8193850 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "os" "git.ohea.xyz/cursorius/server/config" + "git.ohea.xyz/cursorius/server/database" "git.ohea.xyz/cursorius/server/listen" "git.ohea.xyz/cursorius/server/pipeline_api" "git.ohea.xyz/cursorius/server/poll" @@ -31,11 +32,19 @@ func main() { configData, err := config.GetConfig() if err != nil { log.Errorf("Could not get configuration: %v", err) + return + } + + err = database.LaunchDB(configData.Config.DBConfig) + if err != nil { + log.Errorf("Could not launch db: %v", err) + return } getRunnerCh, registerCh, err := runnermanager.StartRunnerManager(configData.Config.Runners) if err != nil { log.Errorf("Could not start runner: %v", err) + return } poll.StartPolling(configData.Config)