Job exection logic now runs pipeline in container
This container's network is configured based on parameters in the config file. If configured correctly, this will allow the pipeline script to speak to the cursorius server over the pipeline api.
This commit is contained in:
+34
-4
@@ -28,20 +28,50 @@ type Runner struct {
|
||||
Secret string
|
||||
}
|
||||
|
||||
type MountType string
|
||||
|
||||
const (
|
||||
Bind MountType = "bind"
|
||||
Volume = "volume"
|
||||
)
|
||||
|
||||
type MountConf struct {
|
||||
Type MountType
|
||||
Source string
|
||||
}
|
||||
|
||||
type PipelineConf struct {
|
||||
AccessURL string // URL that the pipeline runner can access the server at
|
||||
DockerNetwork *string // Name of the docker network that should be assigned to the pipeline script runner
|
||||
WorkingDir string
|
||||
MountConf MountConf // This script describes how to mount WorkingDir into the pipeline executor container
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Address string
|
||||
Port int
|
||||
Jobs map[string]Job
|
||||
Runners map[string]Runner
|
||||
Address string
|
||||
Port int
|
||||
PipelineConf PipelineConf
|
||||
Jobs map[string]Job
|
||||
Runners map[string]Runner
|
||||
}
|
||||
|
||||
func GetConfig() (config.Config[Config], error) {
|
||||
defaultNetworkName := "cursorius"
|
||||
configData := config.Config[Config]{
|
||||
Name: "cursorius",
|
||||
Filename: "server",
|
||||
Config: Config{
|
||||
Address: "127.0.0.1",
|
||||
Port: 45420,
|
||||
PipelineConf: PipelineConf{
|
||||
AccessURL: "cursorius-server:45420",
|
||||
DockerNetwork: &defaultNetworkName,
|
||||
WorkingDir: "/opt/cursorius/working",
|
||||
MountConf: MountConf{
|
||||
Type: Bind,
|
||||
Source: "/opt/cursorius/working",
|
||||
},
|
||||
},
|
||||
Jobs: make(map[string]Job),
|
||||
Runners: make(map[string]Runner),
|
||||
},
|
||||
|
||||
@@ -11,3 +11,8 @@ services:
|
||||
- "../server.toml:/root/.config/cursorius/server.toml"
|
||||
- "/var/run/docker.sock:/var/run/docker.sock"
|
||||
- "../_working/go:/go"
|
||||
- "../_working/jobs:/cursorius/jobs"
|
||||
|
||||
networks:
|
||||
cursorius:
|
||||
external: true
|
||||
|
||||
@@ -5,6 +5,7 @@ go 1.19
|
||||
require (
|
||||
git.ohea.xyz/golang/config v0.0.0-20220915224621-b9debd233173
|
||||
github.com/bufbuild/connect-go v1.4.1
|
||||
github.com/docker/docker v20.10.22+incompatible
|
||||
github.com/go-git/go-git/v5 v5.4.3-0.20220529141257-bc1f419cebcf
|
||||
github.com/go-playground/webhooks/v6 v6.0.1
|
||||
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
|
||||
@@ -19,19 +20,29 @@ require (
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20220930113650-c6815a8c17ad // indirect
|
||||
github.com/acomagu/bufpipe v1.0.3 // indirect
|
||||
github.com/cloudflare/circl v1.2.0 // indirect
|
||||
github.com/docker/distribution v2.8.1+incompatible // indirect
|
||||
github.com/docker/go-connections v0.4.0 // indirect
|
||||
github.com/docker/go-units v0.4.0 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/go-git/gcfg v1.5.0 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.3.1 // indirect
|
||||
github.com/gobwas/ws v1.1.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/imdario/mergo v0.3.13 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
github.com/klauspost/compress v1.15.11 // indirect
|
||||
github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/morikuni/aec v1.0.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/sergi/go-diff v1.2.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.2 // indirect
|
||||
golang.org/x/crypto v0.2.1-0.20221112162523-6fad3dfc1891 // indirect
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
|
||||
@@ -40,6 +51,7 @@ require (
|
||||
golang.org/x/tools v0.1.12 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
gotest.tools/v3 v3.4.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/go-playground/webhooks/v6 => git.ohea.xyz/cursorius/webhooks/v6 v6.0.2-0.20221224221147-a2bdbf1756ed
|
||||
|
||||
@@ -102,6 +102,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJc
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
|
||||
github.com/Azure/azure-service-bus-go v0.9.1/go.mod h1:yzBx6/BUGfjfeqbRZny9AQIbIe3AcV9WZbAdpkoXOa0=
|
||||
github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
|
||||
github.com/Azure/go-autorest v12.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
@@ -347,6 +348,13 @@ github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmW
|
||||
github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
|
||||
github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
|
||||
github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
|
||||
github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
|
||||
github.com/docker/docker v20.10.22+incompatible h1:6jX4yB+NtcbldT90k7vBSaWJDB3i+zkVJT9BEK8kQkk=
|
||||
github.com/docker/docker v20.10.22+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
|
||||
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
|
||||
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
|
||||
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
|
||||
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
|
||||
@@ -542,6 +550,7 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
|
||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
|
||||
github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14/go.mod h1:jPoNZLWDAqA5N3G5amEoiNbhVrmM+ZQEcnQvNQ2KaZk=
|
||||
@@ -1014,6 +1023,8 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
|
||||
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
|
||||
github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
|
||||
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
|
||||
github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA=
|
||||
github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
@@ -1024,6 +1035,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
|
||||
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
|
||||
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
|
||||
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
|
||||
github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
|
||||
github.com/mrjones/oauth v0.0.0-20180629183705-f4e24b6d100c/go.mod h1:skjdDftzkFALcuGzYSklqYd8gvat6F1gZJ4YPVbkZpM=
|
||||
github.com/mrjones/oauth v0.0.0-20190623134757-126b35219450/go.mod h1:skjdDftzkFALcuGzYSklqYd8gvat6F1gZJ4YPVbkZpM=
|
||||
@@ -1076,6 +1089,10 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl
|
||||
github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
|
||||
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
|
||||
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
|
||||
github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
|
||||
github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
|
||||
github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
|
||||
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
@@ -1224,6 +1241,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
|
||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
@@ -1789,6 +1807,7 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb
|
||||
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y=
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@@ -2136,6 +2155,8 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C
|
||||
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
|
||||
gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
||||
+4
-4
@@ -17,11 +17,11 @@ import (
|
||||
var log = logging.MustGetLogger("cursorius-server")
|
||||
|
||||
func setupHTTPServer(runCh chan jobscheduler.Run, registerCh chan jobscheduler.RunnerRegistration,
|
||||
jobs map[string]config.Job) *http.ServeMux {
|
||||
conf config.Config) *http.ServeMux {
|
||||
|
||||
mux := http.NewServeMux()
|
||||
|
||||
webhook.CreateWebhookHandler(runCh, jobs, mux)
|
||||
webhook.CreateWebhookHandler(runCh, conf, mux)
|
||||
pipeline_api.CreateHandler(mux)
|
||||
mux.HandleFunc("/runner", func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := websocket.Accept(w, r, nil)
|
||||
@@ -34,9 +34,9 @@ func setupHTTPServer(runCh chan jobscheduler.Run, registerCh chan jobscheduler.R
|
||||
return mux
|
||||
}
|
||||
|
||||
func Listen(address string, port int, runCh chan jobscheduler.Run, registerCh chan jobscheduler.RunnerRegistration, jobs map[string]config.Job) {
|
||||
func Listen(address string, port int, runCh chan jobscheduler.Run, registerCh chan jobscheduler.RunnerRegistration, conf config.Config) {
|
||||
|
||||
mux := setupHTTPServer(runCh, registerCh, jobs)
|
||||
mux := setupHTTPServer(runCh, registerCh, conf)
|
||||
|
||||
connect_string := fmt.Sprintf("%v:%v", address, port)
|
||||
log.Noticef("Launching HTTP server on %v\n", connect_string)
|
||||
|
||||
@@ -36,8 +36,8 @@ func main() {
|
||||
log.Errorf("Could not start runner: %v", err)
|
||||
}
|
||||
|
||||
poll.StartPolling(configData.Config.Jobs, runCh)
|
||||
poll.StartPolling(configData.Config, runCh)
|
||||
|
||||
listen.Listen(configData.Config.Address, configData.Config.Port, runCh, registerCh, configData.Config.Jobs)
|
||||
listen.Listen(configData.Config.Address, configData.Config.Port, runCh, registerCh, configData.Config)
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
package pipeline_executor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
|
||||
"git.ohea.xyz/cursorius/server/config"
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/container"
|
||||
"github.com/docker/docker/api/types/mount"
|
||||
"github.com/docker/docker/client"
|
||||
"github.com/docker/docker/pkg/stdcopy"
|
||||
"github.com/op/go-logging"
|
||||
)
|
||||
|
||||
var log = logging.MustGetLogger("cursorius-server")
|
||||
|
||||
type PipelineExecution struct {
|
||||
Name string
|
||||
Job config.Job
|
||||
Ref string
|
||||
}
|
||||
|
||||
func ExecutePipeline(pe PipelineExecution, pipelineConf config.PipelineConf) error {
|
||||
jobFolder := filepath.Join(pipelineConf.WorkingDir, pe.Name)
|
||||
|
||||
log.Debugf("Job %v configured with URL \"%v\"", pe.Name, pe.Job.URL)
|
||||
|
||||
log.Debugf("Job %v configured with folder \"%v\"", pe.Name, jobFolder)
|
||||
|
||||
err := os.RemoveAll(jobFolder)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not delete existing folder %v", jobFolder)
|
||||
}
|
||||
|
||||
err = os.MkdirAll(jobFolder, 0755)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create working directory for job %v: %v", pe.Name, err)
|
||||
}
|
||||
|
||||
log.Infof("Cloning source from URL %v", pe.Job.URL)
|
||||
// TODO: should I use go-git here instead of shelling out to raw git?
|
||||
cloneCmd := exec.Command("git", "clone", pe.Job.URL, jobFolder)
|
||||
output, err := cloneCmd.CombinedOutput()
|
||||
if err != nil {
|
||||
log.Debugf("%s", output)
|
||||
return fmt.Errorf("could not clone source: %v", err)
|
||||
}
|
||||
|
||||
cli, err := client.NewClientWithOpts(client.FromEnv)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Could not create docker client: %v", err)
|
||||
}
|
||||
log.Info("Source cloned successfully")
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
imageName := "git.ohea.xyz/cursorius/cursorius:latest"
|
||||
|
||||
log.Infof("Pulling image %v", imageName)
|
||||
pullOutput, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not pull image %v: %v", imageName, err)
|
||||
}
|
||||
|
||||
buf, err := io.ReadAll(pullOutput)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not read from io.ReadCloser:, %v", err)
|
||||
}
|
||||
log.Infof("%s", buf)
|
||||
|
||||
err = pullOutput.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not close io.ReadCloser: %v", err)
|
||||
}
|
||||
log.Info("Image pulled sucessfully")
|
||||
|
||||
hostConfig := container.HostConfig{}
|
||||
|
||||
if pipelineConf.DockerNetwork != nil {
|
||||
hostConfig.NetworkMode = container.NetworkMode(*pipelineConf.DockerNetwork)
|
||||
}
|
||||
|
||||
if pipelineConf.MountConf.Type == config.Bind {
|
||||
hostConfig.Mounts = append(hostConfig.Mounts,
|
||||
mount.Mount{
|
||||
Type: mount.TypeBind,
|
||||
Source: fmt.Sprintf("%v/%v", pipelineConf.MountConf.Source, pe.Name),
|
||||
Target: "/cursorius/src",
|
||||
ReadOnly: false,
|
||||
Consistency: mount.ConsistencyDefault,
|
||||
},
|
||||
)
|
||||
} else if pipelineConf.MountConf.Type == config.Volume {
|
||||
hostConfig.Mounts = append(hostConfig.Mounts,
|
||||
mount.Mount{
|
||||
Type: mount.TypeVolume,
|
||||
Source: pipelineConf.MountConf.Source,
|
||||
Target: "/cursorius/src",
|
||||
ReadOnly: false,
|
||||
Consistency: mount.ConsistencyDefault,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
resp, err := cli.ContainerCreate(ctx,
|
||||
&container.Config{
|
||||
Image: imageName,
|
||||
Cmd: []string{"/launcher.sh"},
|
||||
Tty: false,
|
||||
Env: []string{
|
||||
"CURSORIUS_SRC_DIR=/cursorius/src",
|
||||
fmt.Sprintf("CUROSRIUS_SERVER_URL=%v", pipelineConf.AccessURL),
|
||||
},
|
||||
},
|
||||
// TODO: fix running the runner in docker (add VolumesFrom to HostConfig)
|
||||
&hostConfig,
|
||||
nil, nil, "",
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create container: %v", err)
|
||||
}
|
||||
|
||||
log.Info("Launching container")
|
||||
|
||||
if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
|
||||
return fmt.Errorf("could not start container: %v", err)
|
||||
}
|
||||
statusCh, errCh := cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning)
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if err != nil {
|
||||
return fmt.Errorf("container returned error: %v", err)
|
||||
}
|
||||
case retCode := <-statusCh:
|
||||
log.Debugf("Container finished running with return code: %v", retCode)
|
||||
}
|
||||
|
||||
out, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get container logs: %v", err)
|
||||
}
|
||||
|
||||
var stdOut bytes.Buffer
|
||||
var stdErr bytes.Buffer
|
||||
|
||||
stdcopy.StdCopy(&stdOut, &stdErr, out)
|
||||
|
||||
log.Debugf("%s", stdOut.Bytes())
|
||||
log.Debugf("%s", stdErr.Bytes())
|
||||
|
||||
return nil
|
||||
}
|
||||
+12
-8
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"git.ohea.xyz/cursorius/server/config"
|
||||
"git.ohea.xyz/cursorius/server/jobscheduler"
|
||||
"git.ohea.xyz/cursorius/server/pipeline_executor"
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/storage/memory"
|
||||
@@ -24,7 +25,7 @@ type tag struct {
|
||||
commitHash string
|
||||
}
|
||||
|
||||
func pollJob(repoName string, jobConfig config.Job, runCh chan jobscheduler.Run) {
|
||||
func pollJob(repoName string, jobConfig config.Job, runCh chan jobscheduler.Run, pipelineConf config.PipelineConf) {
|
||||
prevCommits := make(map[string]string)
|
||||
for {
|
||||
time.Sleep(time.Duration(jobConfig.PollInterval) * time.Second)
|
||||
@@ -91,21 +92,24 @@ func pollJob(repoName string, jobConfig config.Job, runCh chan jobscheduler.Run)
|
||||
|
||||
for _, ref := range refsToRunFor {
|
||||
log.Debugf("Dispatching job for ref %v in repo %v", ref, repoName)
|
||||
runCh <- jobscheduler.Run{
|
||||
JobName: repoName,
|
||||
JobConfig: jobConfig,
|
||||
Ref: ref,
|
||||
|
||||
pe := pipeline_executor.PipelineExecution{
|
||||
Name: repoName,
|
||||
Job: jobConfig,
|
||||
Ref: ref,
|
||||
}
|
||||
|
||||
pipeline_executor.ExecutePipeline(pe, pipelineConf)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func StartPolling(jobs map[string]config.Job, runCh chan jobscheduler.Run) {
|
||||
for jobName, job := range jobs {
|
||||
func StartPolling(conf config.Config, runCh chan jobscheduler.Run) {
|
||||
for jobName, job := range conf.Jobs {
|
||||
if job.PollInterval == 0 {
|
||||
continue
|
||||
} else {
|
||||
go pollJob(jobName, job, runCh)
|
||||
go pollJob(jobName, job, runCh, conf.PipelineConf)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+8
-6
@@ -6,13 +6,14 @@ import (
|
||||
|
||||
"git.ohea.xyz/cursorius/server/config"
|
||||
"git.ohea.xyz/cursorius/server/jobscheduler"
|
||||
"git.ohea.xyz/cursorius/server/pipeline_executor"
|
||||
"github.com/go-playground/webhooks/v6/gitea"
|
||||
"github.com/op/go-logging"
|
||||
)
|
||||
|
||||
var log = logging.MustGetLogger("cursorius-server")
|
||||
|
||||
func CreateWebhookHandler(runCh chan jobscheduler.Run, jobs map[string]config.Job, mux *http.ServeMux) {
|
||||
func CreateWebhookHandler(runCh chan jobscheduler.Run, conf config.Config, mux *http.ServeMux) {
|
||||
mux.HandleFunc("/webhook/", func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case "POST":
|
||||
@@ -26,7 +27,7 @@ func CreateWebhookHandler(runCh chan jobscheduler.Run, jobs map[string]config.Jo
|
||||
// TODO: verify that this handles all valid URL formats
|
||||
webhookJobName := splitUrl[2]
|
||||
|
||||
for jobName, jobConfig := range jobs {
|
||||
for jobName, jobConfig := range conf.Jobs {
|
||||
if webhookJobName == jobName {
|
||||
if jobConfig.Webhook == nil {
|
||||
log.Errorf("Matching job does not have webhook configuration, ignoring....")
|
||||
@@ -54,12 +55,13 @@ func CreateWebhookHandler(runCh chan jobscheduler.Run, jobs map[string]config.Jo
|
||||
case gitea.PushPayload:
|
||||
pushPayload := payload.(gitea.PushPayload)
|
||||
|
||||
runCh <- jobscheduler.Run{
|
||||
JobName: jobName,
|
||||
JobConfig: jobConfig,
|
||||
Ref: pushPayload.Ref,
|
||||
pe := pipeline_executor.PipelineExecution{
|
||||
Name: webhookJobName,
|
||||
Job: jobConfig,
|
||||
Ref: pushPayload.Ref,
|
||||
}
|
||||
|
||||
pipeline_executor.ExecutePipeline(pe, conf.PipelineConf)
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user