diff --git a/.dockerfile b/.dockerfile deleted file mode 100644 index b3174e9..0000000 --- a/.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -# TODO: use alpine when available -FROM debian:12-slim -ADD artifacts.tar.gz /install -RUN dpkg -i /install/artifacts/sci_*-1_amd64.deb -RUN rm -rf /install -ENTRYPOINT ["sci"] diff --git a/.sci.sh b/.sci.sh index a358d9a..f9a8444 100755 --- a/.sci.sh +++ b/.sci.sh @@ -2,6 +2,7 @@ set -e echo ">>> checking if required environment is set..." test -n "$DOCKER_TOKEN" +which make echo ">>> compiling..." make @@ -12,7 +13,7 @@ make dist SRC_SHA256=$(sha256sum "sci-${VERSION}.tar.gz" | awk '{ print $1 }') sed "s/SRC_SHA256/${SRC_SHA256}/g" < PKGBUILD.in > PKGBUILD -# arch +# # arch echo ">>> building archbuilder image..." docker build -t archbuilder -f arch-builder.dockerfile . @@ -29,7 +30,7 @@ echo ">>> building debbuilder image..." docker build -t debbuilder -f deb-builder.dockerfile . echo ">>> building .deb in debbuilder docker image..." -docker run --rm -it -v .:/src -e VERSION debbuilder sh -c '\ +docker run --rm -it -v .:/src -e VERSION -e DOCKER_TOKEN debbuilder sh -c '\ cd && \ mkdir -p artifacts && \ cp /src/sci-$VERSION.tar.gz . && \ @@ -46,14 +47,8 @@ docker run --rm -it -v .:/src -e VERSION debbuilder sh -c '\ cp ../*.tar.xz ~/artifacts && \ cp ../*.tar.gz ~/artifacts && \ cd && \ - tar czf /src/artifacts.tar.gz artifacts + curl --user agj:$DOCKER_TOKEN \ + --upload-file sci_$VERSION-1_amd64.deb \ + "https://git.gtz.dk/api/packages/agj/debian/pool/bionic/main/upload" ' - -echo ">>> building sci docker image..." -export OWNER="git.gtz.dk/agj" -docker build -t ${OWNER}/sci:${VERSION} -t ${OWNER}/sci:latest -f .dockerfile . - -echo ">>> pushing latest docker image..." -# TODO: user should be some sci-bot or something, not your account. This will do for now though -docker login git.gtz.dk -u agj -p "$DOCKER_TOKEN" -docker push ${OWNER}/sci:latest +# TODO: push-user should be some sci-bot or something, not your account. This will do for now though diff --git a/Makefile b/Makefile index 4bf0d8d..cabe7ea 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,8 @@ OBJ += out/obj/util.o OBJ += out/obj/pipeline.o OBJ += out/obj/threadlist.o OBJ += out/obj/threadpool.o +OBJ += out/obj/argv_split.o +OBJ += out/obj/which.o out/bin/sci: $(OBJ) | $(BINDIR) $(CC) -o $@ $^ $(CFLAGS) diff --git a/TODO.md b/TODO.md index 2395a32..5414299 100644 --- a/TODO.md +++ b/TODO.md @@ -17,15 +17,17 @@ - [-] ~~alpine~~ later. - [-] ~~docker~~ later. - [ ] Eight things eight, try it out! - maybe even write the python webhook extension. + - [ ] Port this document to gitea issue tracking - [ ] Ninth things ninth, fix bugs, see below - [ ] Tenth things tenth, write manpages, choose license - [ ] Eleventh things Eleventh, polish - [ ] Twelveth things last, release! - - Setup gitea.gtz.dk (will learn you how to set up subdomains (useful for shop.gtz.dk)) + - [x] Setup git.gtz.dk (will learn you how to set up subdomains (useful for shop.gtz.dk)) + - [ ] -1th things -1th, write a blog post about the tool (also set up your blog.gtz.dk) -BOOKMARK: okay. Now it feels like it's getting complicated. I want to run `sci` in a docker container. But that means +Okay. Now it feels like it's getting complicated. I want to run `sci` in a docker container. But that means that the build-threads also run in that docker container - meaning the container should have all the build dependencies -installed and we all know where that rabbithole goes. 9-30YB docker images with about a trillion unique build systems. +installed and we all know where that rabbithole goes. 9-30YiB docker images with about a trillion unique build systems. Let's not do that. The only alternative I can see is that the `sci` service is just not dockerized. The pipeline scripts can easily be dockerized themselves. Just have a `scripts/wget-src-dist-and-sci-sh-dockerized.sh` with `arg1` being the docker image @@ -33,12 +35,24 @@ to use? ```sh #!/bin/sh wget "$SCI_PIPELINE_URL" -docker run --rm -it --mount type=bind,source="$(pwd)"/thefileyouwgot.tar.gz,target=/thefileyouwgot.tar.gz,readonly --entrypoint sh $2 +docker run --rm -it -v .:/src -w /src $@ ``` Or something like that... Perhaps we can figure something out with an inline `ADD`, that also extracts the archive in the container or something. This approach is cleaner IMO. You can also more easily edit the `pipelines.conf` file if you need to. +The aforementioned rabbithole went like this: + - Let's say that `sci` is run inside a docker container. + This would make it very easy to deploy, but: + - Since pipelines are executed in the same environment as `sci`, either: + The `sci` container must be all-encompassing. i.e. it contains every single build system and scriptling language that + could possibly be used by any kind of user or; all pipelines must be run from a docker container themselves, meaning + that the `sci` container must have `dind`-privileges. Either option is suboptimal and will lock users into one way of + using `sci`, which is bad. + - Conclusion: Fuck docker. All environment management is delegated to the user and is not `sci`'s responsibility! + `sci` will always be run on the ci-machine itself, unless a user has provided a custom docker image, which is fine + and doesn't burden the `sci` project. + You were getting the following `pipelines.conf` file to work: ``` scih-dev ssh://git@git.gtz.dk:222/agj/scih.git scih-onpush /etc/sci/scripts/git-clone-and-run-sci-sh.sh @@ -69,7 +83,7 @@ docker is super easy, just make a dockerfile - only concern is the trigger files - [ ] Custom environment variables passed to the pipelines on invokation should be possible. - [ ] Listener threads should be killed and restarted (worker pool should just chug along) when pipeline config file has changed during runtime. Should be disableable with `--no-hot-reload-config` - i.e. on by default. - - [ ] `docker stop` is very slow. I am probably not handling signals properly yet. + - [x] ~~`docker stop` is very slow. I am probably not handling signals properly yet.~~ native docker is abandonned - [x] It seems that `-v 4` is segfaulting when running release builds, maybe the logger just cant find the source file? Nope. I just wrote some bad code (inverted NULL check). diff --git a/include/argv_split.h b/include/argv_split.h new file mode 100644 index 0000000..f8de895 --- /dev/null +++ b/include/argv_split.h @@ -0,0 +1,24 @@ +/** + * sci - a simple ci system + Copyright (C) 2024 Asger Gitz-Johansen + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ +#ifndef SCI_ARGV_SPLIT_H +#define SCI_ARGV_SPLIT_H + +char** create_argv_shell(const char* str, int* argc_out); +void argv_free(char** argv); + +#endif diff --git a/include/which.h b/include/which.h new file mode 100644 index 0000000..c76ac45 --- /dev/null +++ b/include/which.h @@ -0,0 +1,23 @@ +/** + * sci - a simple ci system + Copyright (C) 2024 Asger Gitz-Johansen + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ +#ifndef SCI_WHICH_H +#define SCI_WHICH_H + +int which(const char* program_name, char* out_full_program, int max_path); + +#endif diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..c381a22 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,3 @@ +# sci default scripts +This directory contains some default scripts that may or may not be useful to you. +Most of the scripts are fairly simple, but should be installed as part of the sci installation process. diff --git a/scripts/git-clone-and-run-sci-sh.sh b/scripts/git-clone-and-run-sci-sh.sh deleted file mode 100644 index 9d50675..0000000 --- a/scripts/git-clone-and-run-sci-sh.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -set -e -echo ">>> cloning..." -git clone $SCI_PIPELINE_URL $SCI_PIPELINE_NAME -cd $SCI_PIPELINE_NAME - -echo ">>> running .sci.sh..." -time sh .sci.sh diff --git a/scripts/git-clone-and-sci.sh b/scripts/git-clone-and-sci.sh new file mode 100755 index 0000000..ab08dcc --- /dev/null +++ b/scripts/git-clone-and-sci.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -ex # print all that we're doing (no need for echo's) +tmpdir=$(mktemp -d) +git clone --depth=1 --recurse-submodules --shallow-submodules -b $1 "$SCI_PIPELINE_URL" "$tmpdir" +shift +cd "$tmpdir" +sh .sci.sh +cd - diff --git a/scripts/wget-and-sci.sh b/scripts/wget-and-sci.sh new file mode 100755 index 0000000..4ee671f --- /dev/null +++ b/scripts/wget-and-sci.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# NOTE: This script assumes that the url is a .tar.gz file. +# TODO: check if $# is >= 1 and give a warning that the extract dir should be provided. +set -ex # print all that we're doing (no need for echo's) +tmpdir=$(mktemp -d) +wget "$SCI_PIPELINE_URL" -P "$tmpdir" +cd "$tmpdir" +tar xf *.tar.gz +cd $1 +sh .sci.sh +cd - +rm -rf "$tmpdir" diff --git a/src/argv_split.c b/src/argv_split.c new file mode 100644 index 0000000..66751a3 --- /dev/null +++ b/src/argv_split.c @@ -0,0 +1,81 @@ +/** + * sci - a simple ci system + Copyright (C) 2024 Asger Gitz-Johansen + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ +#include "argv_split.h" +#include +#include +#include +#include + +static const char* skip_arg(const char* cp) { + while(*cp && !isspace(*cp)) + cp++; + return cp; +} + +static char* skip_spaces(const char* str) { + while(isspace(*str)) + str++; + return(char*)str; +} + +static int count_argc(const char* str) { + int count = 0; + while(*str) { + str = skip_spaces(str); + if(!*str) + continue; + count++; + str = skip_arg(str); + } + return count; +} + +void argv_free(char** argv) { + for(char** p = argv; *p; p++) { + free(*p); + *p = NULL; + } + free(argv); +} + +char** create_argv_shell(const char* str, int* argc_out) { + int argc = count_argc(str); + char** result = calloc(argc+1, sizeof(*result)); + if(result == NULL) + return result; + if(argc_out) + *argc_out = argc+1; + char** argvp = result; + while(*str) { + str = skip_spaces(str); + if(!*str) + continue; + const char* p = str; + str = skip_arg(str); + char* t = strndup(p, str-p); + if(t == NULL) { + perror("strndup"); + argv_free(result); + return NULL; + } + *argvp++ = t; + } + *argvp = NULL; + return result; +} +// sci-release https://git.gtz.dk/agj/sci/archive/main.tar.gz manual "scripts/wget-and-sci.sh sci" diff --git a/src/main.c b/src/main.c index a68048b..1833fa0 100644 --- a/src/main.c +++ b/src/main.c @@ -15,16 +15,20 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ +#include "argv_split.h" #include "cli.h" #include "log.h" #include "notify.h" #include "pipeline.h" #include "threadpool.h" #include "util.h" +#include "which.h" #include +#include #include #include #include +#include #include #include @@ -62,8 +66,10 @@ void executor(void* data) { // Create logfile path optional_int fd = open_logfile(pipeline_id); - if(!fd.has_value) + if(!fd.has_value) { + log_error("could not open log file - not starting pipeline"); return; + } // spawn the process pid_t pid; @@ -72,24 +78,35 @@ void executor(void* data) { posix_spawn_file_actions_adddup2(&actions, fd.value, STDOUT_FILENO); posix_spawn_file_actions_adddup2(&actions, fd.value, STDERR_FILENO); const pipeline_event* const e = data; + char* path = join("PATH=", getenv("PATH")); char* name = join("SCI_PIPELINE_NAME=", e->name); char* url = join("SCI_PIPELINE_URL=", e->url); char* trigger = join("SCI_PIPELINE_TRIGGER=", e->trigger); char* id = join("SCI_PIPELINE_ID=", pipeline_id); - char* envp[] = { name, url, trigger, id, NULL }; - char* argv[] = { "/bin/sh", "-c", e->command, NULL }; - if(posix_spawn(&pid, "/bin/sh", &actions, NULL, argv, envp) != 0) { + char* envp[] = { path, name, url, trigger, id, NULL }; + int argc; + char** argv = create_argv_shell(e->command, &argc); + log_trace("executing pipeline %s with argv:", e->name); + for(int i = 0; i < argc; i++) + log_trace(" \"%s\"", argv[i]); + char arg0[PATH_MAX]; + if(which(argv[0], arg0, PATH_MAX) == -1) + goto end; + if(posix_spawn(&pid, arg0, &actions, NULL, argv, envp) != 0) { perror("posix_spawn"); - goto end; // I know. The raptors have picked up the scent. I'll just have to mask it with more poopy code. + goto end; // I know. The raptors have picked up the scent. I'll just have to mask it with more stinky code. } - log_trace("{%s} (%s) spawned", pipeline_id, e->name); + log_info("{%s} (%s) spawned", pipeline_id, e->name); // Wait for process to complete int status; waitpid(pid, &status, 0); - if(WIFEXITED(status)) - log_trace("{%s} (%s) exited with status %d", pipeline_id, e->name, WEXITSTATUS(status)); + log_info("{%s} (%s) exited with status %d", pipeline_id, e->name, status); + char buf[32]; + sprintf(buf, "exited with status %d", status); + write(fd.value, buf, strnlen(buf, 32)); end: + argv_free(argv); close(fd.value); free(pipeline_id); free(name); diff --git a/src/pipeline.c b/src/pipeline.c index cc2feef..4e3f131 100644 --- a/src/pipeline.c +++ b/src/pipeline.c @@ -40,7 +40,11 @@ optional_pipeline_conf pipeline_create(const char* config_line) { break; off = pmatch[0].rm_so + (cursor - config_line); len = pmatch[0].rm_eo - pmatch[0].rm_so; - opts[i] = strndup(config_line + off, len); + // Cut off the "-s if it is string-enclosed + if(config_line[off] == '"' && config_line[off+len-1] == '"') + opts[i] = strndup(config_line + off+1, len-2); + else + opts[i] = strndup(config_line + off, len); cursor += pmatch[0].rm_eo; } if(i != 4) { diff --git a/src/which.c b/src/which.c new file mode 100644 index 0000000..7e7c8b4 --- /dev/null +++ b/src/which.c @@ -0,0 +1,54 @@ +/** + * sci - a simple ci system + Copyright (C) 2024 Asger Gitz-Johansen + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ +#include "log.h" +#include +#include +#include +#include +#include +#include + +int which(const char* program_name, char* out_full_program, int max_path) { + assert(out_full_program); + assert(max_path > 0); + // sanity check - maybe program_name is actually a full-path to begin with + if(access(program_name, X_OK) == 0) { + snprintf(out_full_program, max_path, "%s", program_name); + return 0; + } + char* path = getenv("PATH"); + if (path == NULL) { + log_error("PATH environment variable not found."); + return -1; + } + char* path_cpy = strdup(path); + char* dir = strtok(path_cpy, ":"); + char full_path[PATH_MAX]; + while(dir != NULL) { + snprintf(full_path, sizeof(full_path), "%s/%s", dir, program_name); + if(access(full_path, X_OK) == 0) { + snprintf(out_full_program, max_path, "%s", full_path); + free(path_cpy); + return 0; + } + dir = strtok(NULL, ":"); + } + log_error("'%s' not found in PATH", program_name); + free(path_cpy); + return -1; +}