From 8e4e45047db7f30c8cf087453ec64a2d0cb20353 Mon Sep 17 00:00:00 2001 From: restitux Date: Sat, 24 Dec 2022 22:12:30 -0700 Subject: [PATCH] Job exection logic now runs pipeline in container This container's network is configured based on parameters in the config file. If configured correctly, this will allow the pipeline script to speak to the cursorius server over the pipeline api. --- config/config.go | 38 +++++- docker/docker-compose.yml | 5 + go.mod | 12 ++ go.sum | 21 ++++ listen/listen.go | 8 +- main.go | 4 +- pipeline_executor/pipeline_executor.go | 158 +++++++++++++++++++++++++ poll/poll.go | 20 ++-- webhook/webhook.go | 14 ++- 9 files changed, 256 insertions(+), 24 deletions(-) create mode 100644 pipeline_executor/pipeline_executor.go diff --git a/config/config.go b/config/config.go index 36ab66c..5e3d82f 100644 --- a/config/config.go +++ b/config/config.go @@ -28,20 +28,50 @@ type Runner struct { Secret string } +type MountType string + +const ( + Bind MountType = "bind" + Volume = "volume" +) + +type MountConf struct { + Type MountType + Source string +} + +type PipelineConf struct { + AccessURL string // URL that the pipeline runner can access the server at + DockerNetwork *string // Name of the docker network that should be assigned to the pipeline script runner + WorkingDir string + MountConf MountConf // This script describes how to mount WorkingDir into the pipeline executor container +} + type Config struct { - Address string - Port int - Jobs map[string]Job - Runners map[string]Runner + Address string + Port int + PipelineConf PipelineConf + Jobs map[string]Job + Runners map[string]Runner } func GetConfig() (config.Config[Config], error) { + defaultNetworkName := "cursorius" configData := config.Config[Config]{ Name: "cursorius", Filename: "server", Config: Config{ Address: "127.0.0.1", Port: 45420, + PipelineConf: PipelineConf{ + AccessURL: "cursorius-server:45420", + DockerNetwork: &defaultNetworkName, + WorkingDir: "/opt/cursorius/working", + MountConf: MountConf{ + Type: Bind, + Source: "/opt/cursorius/working", + }, + }, Jobs: make(map[string]Job), Runners: make(map[string]Runner), }, diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 8c0264d..73b0c7b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -11,3 +11,8 @@ services: - "../server.toml:/root/.config/cursorius/server.toml" - "/var/run/docker.sock:/var/run/docker.sock" - "../_working/go:/go" + - "../_working/jobs:/cursorius/jobs" + +networks: + cursorius: + external: true diff --git a/go.mod b/go.mod index 9eb958b..8026831 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( git.ohea.xyz/golang/config v0.0.0-20220915224621-b9debd233173 github.com/bufbuild/connect-go v1.4.1 + github.com/docker/docker v20.10.22+incompatible github.com/go-git/go-git/v5 v5.4.3-0.20220529141257-bc1f419cebcf github.com/go-playground/webhooks/v6 v6.0.1 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 @@ -19,19 +20,29 @@ require ( github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad // indirect github.com/acomagu/bufpipe v1.0.3 // indirect github.com/cloudflare/circl v1.2.0 // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.4.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/gobwas/ws v1.1.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.15.11 // indirect + github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.5 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/sergi/go-diff v1.2.0 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect github.com/xanzy/ssh-agent v0.3.2 // indirect golang.org/x/crypto v0.2.1-0.20221112162523-6fad3dfc1891 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect @@ -40,6 +51,7 @@ require ( golang.org/x/tools v0.1.12 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gotest.tools/v3 v3.4.0 // indirect ) replace github.com/go-playground/webhooks/v6 => git.ohea.xyz/cursorius/webhooks/v6 v6.0.2-0.20221224221147-a2bdbf1756ed diff --git a/go.sum b/go.sum index cd8271e..6b43f9c 100644 --- a/go.sum +++ b/go.sum @@ -102,6 +102,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJc github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= github.com/Azure/azure-service-bus-go v0.9.1/go.mod h1:yzBx6/BUGfjfeqbRZny9AQIbIe3AcV9WZbAdpkoXOa0= github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-autorest v12.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -347,6 +348,13 @@ github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmW github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v20.10.22+incompatible h1:6jX4yB+NtcbldT90k7vBSaWJDB3i+zkVJT9BEK8kQkk= +github.com/docker/docker v20.10.22+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= @@ -542,6 +550,7 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14/go.mod h1:jPoNZLWDAqA5N3G5amEoiNbhVrmM+ZQEcnQvNQ2KaZk= @@ -1014,6 +1023,8 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1024,6 +1035,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/mrjones/oauth v0.0.0-20180629183705-f4e24b6d100c/go.mod h1:skjdDftzkFALcuGzYSklqYd8gvat6F1gZJ4YPVbkZpM= github.com/mrjones/oauth v0.0.0-20190623134757-126b35219450/go.mod h1:skjdDftzkFALcuGzYSklqYd8gvat6F1gZJ4YPVbkZpM= @@ -1076,6 +1089,10 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -1224,6 +1241,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -1789,6 +1807,7 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2136,6 +2155,8 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= +gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/listen/listen.go b/listen/listen.go index 2e5e06d..354591c 100644 --- a/listen/listen.go +++ b/listen/listen.go @@ -17,11 +17,11 @@ import ( var log = logging.MustGetLogger("cursorius-server") func setupHTTPServer(runCh chan jobscheduler.Run, registerCh chan jobscheduler.RunnerRegistration, - jobs map[string]config.Job) *http.ServeMux { + conf config.Config) *http.ServeMux { mux := http.NewServeMux() - webhook.CreateWebhookHandler(runCh, jobs, mux) + webhook.CreateWebhookHandler(runCh, conf, mux) pipeline_api.CreateHandler(mux) mux.HandleFunc("/runner", func(w http.ResponseWriter, r *http.Request) { conn, err := websocket.Accept(w, r, nil) @@ -34,9 +34,9 @@ func setupHTTPServer(runCh chan jobscheduler.Run, registerCh chan jobscheduler.R return mux } -func Listen(address string, port int, runCh chan jobscheduler.Run, registerCh chan jobscheduler.RunnerRegistration, jobs map[string]config.Job) { +func Listen(address string, port int, runCh chan jobscheduler.Run, registerCh chan jobscheduler.RunnerRegistration, conf config.Config) { - mux := setupHTTPServer(runCh, registerCh, jobs) + mux := setupHTTPServer(runCh, registerCh, conf) connect_string := fmt.Sprintf("%v:%v", address, port) log.Noticef("Launching HTTP server on %v\n", connect_string) diff --git a/main.go b/main.go index 77a8ec3..cafd0ef 100644 --- a/main.go +++ b/main.go @@ -36,8 +36,8 @@ func main() { log.Errorf("Could not start runner: %v", err) } - poll.StartPolling(configData.Config.Jobs, runCh) + poll.StartPolling(configData.Config, runCh) - listen.Listen(configData.Config.Address, configData.Config.Port, runCh, registerCh, configData.Config.Jobs) + listen.Listen(configData.Config.Address, configData.Config.Port, runCh, registerCh, configData.Config) } diff --git a/pipeline_executor/pipeline_executor.go b/pipeline_executor/pipeline_executor.go new file mode 100644 index 0000000..3ce7387 --- /dev/null +++ b/pipeline_executor/pipeline_executor.go @@ -0,0 +1,158 @@ +package pipeline_executor + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + + "git.ohea.xyz/cursorius/server/config" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/client" + "github.com/docker/docker/pkg/stdcopy" + "github.com/op/go-logging" +) + +var log = logging.MustGetLogger("cursorius-server") + +type PipelineExecution struct { + Name string + Job config.Job + Ref string +} + +func ExecutePipeline(pe PipelineExecution, pipelineConf config.PipelineConf) error { + jobFolder := filepath.Join(pipelineConf.WorkingDir, pe.Name) + + log.Debugf("Job %v configured with URL \"%v\"", pe.Name, pe.Job.URL) + + log.Debugf("Job %v configured with folder \"%v\"", pe.Name, jobFolder) + + err := os.RemoveAll(jobFolder) + if err != nil { + return fmt.Errorf("could not delete existing folder %v", jobFolder) + } + + err = os.MkdirAll(jobFolder, 0755) + if err != nil { + return fmt.Errorf("could not create working directory for job %v: %v", pe.Name, err) + } + + log.Infof("Cloning source from URL %v", pe.Job.URL) + // TODO: should I use go-git here instead of shelling out to raw git? + cloneCmd := exec.Command("git", "clone", pe.Job.URL, jobFolder) + output, err := cloneCmd.CombinedOutput() + if err != nil { + log.Debugf("%s", output) + return fmt.Errorf("could not clone source: %v", err) + } + + cli, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + return fmt.Errorf("Could not create docker client: %v", err) + } + log.Info("Source cloned successfully") + + ctx := context.Background() + + imageName := "git.ohea.xyz/cursorius/cursorius:latest" + + log.Infof("Pulling image %v", imageName) + pullOutput, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{}) + if err != nil { + return fmt.Errorf("could not pull image %v: %v", imageName, err) + } + + buf, err := io.ReadAll(pullOutput) + if err != nil { + return fmt.Errorf("could not read from io.ReadCloser:, %v", err) + } + log.Infof("%s", buf) + + err = pullOutput.Close() + if err != nil { + return fmt.Errorf("could not close io.ReadCloser: %v", err) + } + log.Info("Image pulled sucessfully") + + hostConfig := container.HostConfig{} + + if pipelineConf.DockerNetwork != nil { + hostConfig.NetworkMode = container.NetworkMode(*pipelineConf.DockerNetwork) + } + + if pipelineConf.MountConf.Type == config.Bind { + hostConfig.Mounts = append(hostConfig.Mounts, + mount.Mount{ + Type: mount.TypeBind, + Source: fmt.Sprintf("%v/%v", pipelineConf.MountConf.Source, pe.Name), + Target: "/cursorius/src", + ReadOnly: false, + Consistency: mount.ConsistencyDefault, + }, + ) + } else if pipelineConf.MountConf.Type == config.Volume { + hostConfig.Mounts = append(hostConfig.Mounts, + mount.Mount{ + Type: mount.TypeVolume, + Source: pipelineConf.MountConf.Source, + Target: "/cursorius/src", + ReadOnly: false, + Consistency: mount.ConsistencyDefault, + }, + ) + } + + resp, err := cli.ContainerCreate(ctx, + &container.Config{ + Image: imageName, + Cmd: []string{"/launcher.sh"}, + Tty: false, + Env: []string{ + "CURSORIUS_SRC_DIR=/cursorius/src", + fmt.Sprintf("CUROSRIUS_SERVER_URL=%v", pipelineConf.AccessURL), + }, + }, + // TODO: fix running the runner in docker (add VolumesFrom to HostConfig) + &hostConfig, + nil, nil, "", + ) + if err != nil { + return fmt.Errorf("could not create container: %v", err) + } + + log.Info("Launching container") + + if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { + return fmt.Errorf("could not start container: %v", err) + } + statusCh, errCh := cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) + select { + case err := <-errCh: + if err != nil { + return fmt.Errorf("container returned error: %v", err) + } + case retCode := <-statusCh: + log.Debugf("Container finished running with return code: %v", retCode) + } + + out, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true}) + if err != nil { + return fmt.Errorf("could not get container logs: %v", err) + } + + var stdOut bytes.Buffer + var stdErr bytes.Buffer + + stdcopy.StdCopy(&stdOut, &stdErr, out) + + log.Debugf("%s", stdOut.Bytes()) + log.Debugf("%s", stdErr.Bytes()) + + return nil +} diff --git a/poll/poll.go b/poll/poll.go index 96ccb82..73b51f0 100644 --- a/poll/poll.go +++ b/poll/poll.go @@ -7,6 +7,7 @@ import ( "git.ohea.xyz/cursorius/server/config" "git.ohea.xyz/cursorius/server/jobscheduler" + "git.ohea.xyz/cursorius/server/pipeline_executor" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" @@ -24,7 +25,7 @@ type tag struct { commitHash string } -func pollJob(repoName string, jobConfig config.Job, runCh chan jobscheduler.Run) { +func pollJob(repoName string, jobConfig config.Job, runCh chan jobscheduler.Run, pipelineConf config.PipelineConf) { prevCommits := make(map[string]string) for { time.Sleep(time.Duration(jobConfig.PollInterval) * time.Second) @@ -91,21 +92,24 @@ func pollJob(repoName string, jobConfig config.Job, runCh chan jobscheduler.Run) for _, ref := range refsToRunFor { log.Debugf("Dispatching job for ref %v in repo %v", ref, repoName) - runCh <- jobscheduler.Run{ - JobName: repoName, - JobConfig: jobConfig, - Ref: ref, + + pe := pipeline_executor.PipelineExecution{ + Name: repoName, + Job: jobConfig, + Ref: ref, } + + pipeline_executor.ExecutePipeline(pe, pipelineConf) } } } -func StartPolling(jobs map[string]config.Job, runCh chan jobscheduler.Run) { - for jobName, job := range jobs { +func StartPolling(conf config.Config, runCh chan jobscheduler.Run) { + for jobName, job := range conf.Jobs { if job.PollInterval == 0 { continue } else { - go pollJob(jobName, job, runCh) + go pollJob(jobName, job, runCh, conf.PipelineConf) } } } diff --git a/webhook/webhook.go b/webhook/webhook.go index 5459700..502751a 100644 --- a/webhook/webhook.go +++ b/webhook/webhook.go @@ -6,13 +6,14 @@ import ( "git.ohea.xyz/cursorius/server/config" "git.ohea.xyz/cursorius/server/jobscheduler" + "git.ohea.xyz/cursorius/server/pipeline_executor" "github.com/go-playground/webhooks/v6/gitea" "github.com/op/go-logging" ) var log = logging.MustGetLogger("cursorius-server") -func CreateWebhookHandler(runCh chan jobscheduler.Run, jobs map[string]config.Job, mux *http.ServeMux) { +func CreateWebhookHandler(runCh chan jobscheduler.Run, conf config.Config, mux *http.ServeMux) { mux.HandleFunc("/webhook/", func(w http.ResponseWriter, r *http.Request) { switch r.Method { case "POST": @@ -26,7 +27,7 @@ func CreateWebhookHandler(runCh chan jobscheduler.Run, jobs map[string]config.Jo // TODO: verify that this handles all valid URL formats webhookJobName := splitUrl[2] - for jobName, jobConfig := range jobs { + for jobName, jobConfig := range conf.Jobs { if webhookJobName == jobName { if jobConfig.Webhook == nil { log.Errorf("Matching job does not have webhook configuration, ignoring....") @@ -54,12 +55,13 @@ func CreateWebhookHandler(runCh chan jobscheduler.Run, jobs map[string]config.Jo case gitea.PushPayload: pushPayload := payload.(gitea.PushPayload) - runCh <- jobscheduler.Run{ - JobName: jobName, - JobConfig: jobConfig, - Ref: pushPayload.Ref, + pe := pipeline_executor.PipelineExecution{ + Name: webhookJobName, + Job: jobConfig, + Ref: pushPayload.Ref, } + pipeline_executor.ExecutePipeline(pe, conf.PipelineConf) } return