diff --git a/config/config.go b/config/config.go index 36ab66c..5e3d82f 100644 --- a/config/config.go +++ b/config/config.go @@ -28,20 +28,50 @@ type Runner struct { Secret string } +type MountType string + +const ( + Bind MountType = "bind" + Volume = "volume" +) + +type MountConf struct { + Type MountType + Source string +} + +type PipelineConf struct { + AccessURL string // URL that the pipeline runner can access the server at + DockerNetwork *string // Name of the docker network that should be assigned to the pipeline script runner + WorkingDir string + MountConf MountConf // This script describes how to mount WorkingDir into the pipeline executor container +} + type Config struct { - Address string - Port int - Jobs map[string]Job - Runners map[string]Runner + Address string + Port int + PipelineConf PipelineConf + Jobs map[string]Job + Runners map[string]Runner } func GetConfig() (config.Config[Config], error) { + defaultNetworkName := "cursorius" configData := config.Config[Config]{ Name: "cursorius", Filename: "server", Config: Config{ Address: "127.0.0.1", Port: 45420, + PipelineConf: PipelineConf{ + AccessURL: "cursorius-server:45420", + DockerNetwork: &defaultNetworkName, + WorkingDir: "/opt/cursorius/working", + MountConf: MountConf{ + Type: Bind, + Source: "/opt/cursorius/working", + }, + }, Jobs: make(map[string]Job), Runners: make(map[string]Runner), }, diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 8c0264d..73b0c7b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -11,3 +11,8 @@ services: - "../server.toml:/root/.config/cursorius/server.toml" - "/var/run/docker.sock:/var/run/docker.sock" - "../_working/go:/go" + - "../_working/jobs:/cursorius/jobs" + +networks: + cursorius: + external: true diff --git a/go.mod b/go.mod index 9eb958b..8026831 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( git.ohea.xyz/golang/config v0.0.0-20220915224621-b9debd233173 github.com/bufbuild/connect-go v1.4.1 + github.com/docker/docker v20.10.22+incompatible github.com/go-git/go-git/v5 v5.4.3-0.20220529141257-bc1f419cebcf github.com/go-playground/webhooks/v6 v6.0.1 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 @@ -19,19 +20,29 @@ require ( github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad // indirect github.com/acomagu/bufpipe v1.0.3 // indirect github.com/cloudflare/circl v1.2.0 // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.4.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/gobwas/ws v1.1.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.15.11 // indirect + github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.5 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/sergi/go-diff v1.2.0 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect github.com/xanzy/ssh-agent v0.3.2 // indirect golang.org/x/crypto v0.2.1-0.20221112162523-6fad3dfc1891 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect @@ -40,6 +51,7 @@ require ( golang.org/x/tools v0.1.12 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gotest.tools/v3 v3.4.0 // indirect ) replace github.com/go-playground/webhooks/v6 => git.ohea.xyz/cursorius/webhooks/v6 v6.0.2-0.20221224221147-a2bdbf1756ed diff --git a/go.sum b/go.sum index cd8271e..6b43f9c 100644 --- a/go.sum +++ b/go.sum @@ -102,6 +102,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJc github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= github.com/Azure/azure-service-bus-go v0.9.1/go.mod h1:yzBx6/BUGfjfeqbRZny9AQIbIe3AcV9WZbAdpkoXOa0= github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-autorest v12.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -347,6 +348,13 @@ github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmW github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v20.10.22+incompatible h1:6jX4yB+NtcbldT90k7vBSaWJDB3i+zkVJT9BEK8kQkk= +github.com/docker/docker v20.10.22+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= @@ -542,6 +550,7 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14/go.mod h1:jPoNZLWDAqA5N3G5amEoiNbhVrmM+ZQEcnQvNQ2KaZk= @@ -1014,6 +1023,8 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1024,6 +1035,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/mrjones/oauth v0.0.0-20180629183705-f4e24b6d100c/go.mod h1:skjdDftzkFALcuGzYSklqYd8gvat6F1gZJ4YPVbkZpM= github.com/mrjones/oauth v0.0.0-20190623134757-126b35219450/go.mod h1:skjdDftzkFALcuGzYSklqYd8gvat6F1gZJ4YPVbkZpM= @@ -1076,6 +1089,10 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -1224,6 +1241,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -1789,6 +1807,7 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2136,6 +2155,8 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= +gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/listen/listen.go b/listen/listen.go index 2e5e06d..354591c 100644 --- a/listen/listen.go +++ b/listen/listen.go @@ -17,11 +17,11 @@ import ( var log = logging.MustGetLogger("cursorius-server") func setupHTTPServer(runCh chan jobscheduler.Run, registerCh chan jobscheduler.RunnerRegistration, - jobs map[string]config.Job) *http.ServeMux { + conf config.Config) *http.ServeMux { mux := http.NewServeMux() - webhook.CreateWebhookHandler(runCh, jobs, mux) + webhook.CreateWebhookHandler(runCh, conf, mux) pipeline_api.CreateHandler(mux) mux.HandleFunc("/runner", func(w http.ResponseWriter, r *http.Request) { conn, err := websocket.Accept(w, r, nil) @@ -34,9 +34,9 @@ func setupHTTPServer(runCh chan jobscheduler.Run, registerCh chan jobscheduler.R return mux } -func Listen(address string, port int, runCh chan jobscheduler.Run, registerCh chan jobscheduler.RunnerRegistration, jobs map[string]config.Job) { +func Listen(address string, port int, runCh chan jobscheduler.Run, registerCh chan jobscheduler.RunnerRegistration, conf config.Config) { - mux := setupHTTPServer(runCh, registerCh, jobs) + mux := setupHTTPServer(runCh, registerCh, conf) connect_string := fmt.Sprintf("%v:%v", address, port) log.Noticef("Launching HTTP server on %v\n", connect_string) diff --git a/main.go b/main.go index 77a8ec3..cafd0ef 100644 --- a/main.go +++ b/main.go @@ -36,8 +36,8 @@ func main() { log.Errorf("Could not start runner: %v", err) } - poll.StartPolling(configData.Config.Jobs, runCh) + poll.StartPolling(configData.Config, runCh) - listen.Listen(configData.Config.Address, configData.Config.Port, runCh, registerCh, configData.Config.Jobs) + listen.Listen(configData.Config.Address, configData.Config.Port, runCh, registerCh, configData.Config) } diff --git a/pipeline_executor/pipeline_executor.go b/pipeline_executor/pipeline_executor.go new file mode 100644 index 0000000..3ce7387 --- /dev/null +++ b/pipeline_executor/pipeline_executor.go @@ -0,0 +1,158 @@ +package pipeline_executor + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + + "git.ohea.xyz/cursorius/server/config" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/client" + "github.com/docker/docker/pkg/stdcopy" + "github.com/op/go-logging" +) + +var log = logging.MustGetLogger("cursorius-server") + +type PipelineExecution struct { + Name string + Job config.Job + Ref string +} + +func ExecutePipeline(pe PipelineExecution, pipelineConf config.PipelineConf) error { + jobFolder := filepath.Join(pipelineConf.WorkingDir, pe.Name) + + log.Debugf("Job %v configured with URL \"%v\"", pe.Name, pe.Job.URL) + + log.Debugf("Job %v configured with folder \"%v\"", pe.Name, jobFolder) + + err := os.RemoveAll(jobFolder) + if err != nil { + return fmt.Errorf("could not delete existing folder %v", jobFolder) + } + + err = os.MkdirAll(jobFolder, 0755) + if err != nil { + return fmt.Errorf("could not create working directory for job %v: %v", pe.Name, err) + } + + log.Infof("Cloning source from URL %v", pe.Job.URL) + // TODO: should I use go-git here instead of shelling out to raw git? + cloneCmd := exec.Command("git", "clone", pe.Job.URL, jobFolder) + output, err := cloneCmd.CombinedOutput() + if err != nil { + log.Debugf("%s", output) + return fmt.Errorf("could not clone source: %v", err) + } + + cli, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + return fmt.Errorf("Could not create docker client: %v", err) + } + log.Info("Source cloned successfully") + + ctx := context.Background() + + imageName := "git.ohea.xyz/cursorius/cursorius:latest" + + log.Infof("Pulling image %v", imageName) + pullOutput, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{}) + if err != nil { + return fmt.Errorf("could not pull image %v: %v", imageName, err) + } + + buf, err := io.ReadAll(pullOutput) + if err != nil { + return fmt.Errorf("could not read from io.ReadCloser:, %v", err) + } + log.Infof("%s", buf) + + err = pullOutput.Close() + if err != nil { + return fmt.Errorf("could not close io.ReadCloser: %v", err) + } + log.Info("Image pulled sucessfully") + + hostConfig := container.HostConfig{} + + if pipelineConf.DockerNetwork != nil { + hostConfig.NetworkMode = container.NetworkMode(*pipelineConf.DockerNetwork) + } + + if pipelineConf.MountConf.Type == config.Bind { + hostConfig.Mounts = append(hostConfig.Mounts, + mount.Mount{ + Type: mount.TypeBind, + Source: fmt.Sprintf("%v/%v", pipelineConf.MountConf.Source, pe.Name), + Target: "/cursorius/src", + ReadOnly: false, + Consistency: mount.ConsistencyDefault, + }, + ) + } else if pipelineConf.MountConf.Type == config.Volume { + hostConfig.Mounts = append(hostConfig.Mounts, + mount.Mount{ + Type: mount.TypeVolume, + Source: pipelineConf.MountConf.Source, + Target: "/cursorius/src", + ReadOnly: false, + Consistency: mount.ConsistencyDefault, + }, + ) + } + + resp, err := cli.ContainerCreate(ctx, + &container.Config{ + Image: imageName, + Cmd: []string{"/launcher.sh"}, + Tty: false, + Env: []string{ + "CURSORIUS_SRC_DIR=/cursorius/src", + fmt.Sprintf("CUROSRIUS_SERVER_URL=%v", pipelineConf.AccessURL), + }, + }, + // TODO: fix running the runner in docker (add VolumesFrom to HostConfig) + &hostConfig, + nil, nil, "", + ) + if err != nil { + return fmt.Errorf("could not create container: %v", err) + } + + log.Info("Launching container") + + if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { + return fmt.Errorf("could not start container: %v", err) + } + statusCh, errCh := cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) + select { + case err := <-errCh: + if err != nil { + return fmt.Errorf("container returned error: %v", err) + } + case retCode := <-statusCh: + log.Debugf("Container finished running with return code: %v", retCode) + } + + out, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true}) + if err != nil { + return fmt.Errorf("could not get container logs: %v", err) + } + + var stdOut bytes.Buffer + var stdErr bytes.Buffer + + stdcopy.StdCopy(&stdOut, &stdErr, out) + + log.Debugf("%s", stdOut.Bytes()) + log.Debugf("%s", stdErr.Bytes()) + + return nil +} diff --git a/poll/poll.go b/poll/poll.go index 96ccb82..73b51f0 100644 --- a/poll/poll.go +++ b/poll/poll.go @@ -7,6 +7,7 @@ import ( "git.ohea.xyz/cursorius/server/config" "git.ohea.xyz/cursorius/server/jobscheduler" + "git.ohea.xyz/cursorius/server/pipeline_executor" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" @@ -24,7 +25,7 @@ type tag struct { commitHash string } -func pollJob(repoName string, jobConfig config.Job, runCh chan jobscheduler.Run) { +func pollJob(repoName string, jobConfig config.Job, runCh chan jobscheduler.Run, pipelineConf config.PipelineConf) { prevCommits := make(map[string]string) for { time.Sleep(time.Duration(jobConfig.PollInterval) * time.Second) @@ -91,21 +92,24 @@ func pollJob(repoName string, jobConfig config.Job, runCh chan jobscheduler.Run) for _, ref := range refsToRunFor { log.Debugf("Dispatching job for ref %v in repo %v", ref, repoName) - runCh <- jobscheduler.Run{ - JobName: repoName, - JobConfig: jobConfig, - Ref: ref, + + pe := pipeline_executor.PipelineExecution{ + Name: repoName, + Job: jobConfig, + Ref: ref, } + + pipeline_executor.ExecutePipeline(pe, pipelineConf) } } } -func StartPolling(jobs map[string]config.Job, runCh chan jobscheduler.Run) { - for jobName, job := range jobs { +func StartPolling(conf config.Config, runCh chan jobscheduler.Run) { + for jobName, job := range conf.Jobs { if job.PollInterval == 0 { continue } else { - go pollJob(jobName, job, runCh) + go pollJob(jobName, job, runCh, conf.PipelineConf) } } } diff --git a/webhook/webhook.go b/webhook/webhook.go index 5459700..502751a 100644 --- a/webhook/webhook.go +++ b/webhook/webhook.go @@ -6,13 +6,14 @@ import ( "git.ohea.xyz/cursorius/server/config" "git.ohea.xyz/cursorius/server/jobscheduler" + "git.ohea.xyz/cursorius/server/pipeline_executor" "github.com/go-playground/webhooks/v6/gitea" "github.com/op/go-logging" ) var log = logging.MustGetLogger("cursorius-server") -func CreateWebhookHandler(runCh chan jobscheduler.Run, jobs map[string]config.Job, mux *http.ServeMux) { +func CreateWebhookHandler(runCh chan jobscheduler.Run, conf config.Config, mux *http.ServeMux) { mux.HandleFunc("/webhook/", func(w http.ResponseWriter, r *http.Request) { switch r.Method { case "POST": @@ -26,7 +27,7 @@ func CreateWebhookHandler(runCh chan jobscheduler.Run, jobs map[string]config.Jo // TODO: verify that this handles all valid URL formats webhookJobName := splitUrl[2] - for jobName, jobConfig := range jobs { + for jobName, jobConfig := range conf.Jobs { if webhookJobName == jobName { if jobConfig.Webhook == nil { log.Errorf("Matching job does not have webhook configuration, ignoring....") @@ -54,12 +55,13 @@ func CreateWebhookHandler(runCh chan jobscheduler.Run, jobs map[string]config.Jo case gitea.PushPayload: pushPayload := payload.(gitea.PushPayload) - runCh <- jobscheduler.Run{ - JobName: jobName, - JobConfig: jobConfig, - Ref: pushPayload.Ref, + pe := pipeline_executor.PipelineExecution{ + Name: webhookJobName, + Job: jobConfig, + Ref: pushPayload.Ref, } + pipeline_executor.ExecutePipeline(pe, conf.PipelineConf) } return