From faf362c6074d1f90c9e5dbd5141a524e42d1c214 Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Sat, 24 Aug 2024 11:13:06 +0200 Subject: [PATCH 01/18] feat: add better shell-like command support You can now execute any kind of program in the PATH. You do, however, need to specify "./" if you want to execute a local file, but that shouldn't be too big a problem. --- .dockerfile | 6 --- .sci.sh | 19 +++---- Makefile | 2 + TODO.md | 24 +++++++-- include/argv_split.h | 24 +++++++++ include/which.h | 23 ++++++++ scripts/README.md | 3 ++ scripts/git-clone-and-run-sci-sh.sh | 8 --- scripts/git-clone-and-sci.sh | 8 +++ scripts/wget-and-sci.sh | 12 +++++ src/argv_split.c | 81 +++++++++++++++++++++++++++++ src/main.c | 33 +++++++++--- src/pipeline.c | 6 ++- src/which.c | 54 +++++++++++++++++++ 14 files changed, 263 insertions(+), 40 deletions(-) delete mode 100644 .dockerfile create mode 100644 include/argv_split.h create mode 100644 include/which.h create mode 100644 scripts/README.md delete mode 100644 scripts/git-clone-and-run-sci-sh.sh create mode 100755 scripts/git-clone-and-sci.sh create mode 100755 scripts/wget-and-sci.sh create mode 100644 src/argv_split.c create mode 100644 src/which.c diff --git a/.dockerfile b/.dockerfile deleted file mode 100644 index b3174e9..0000000 --- a/.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -# TODO: use alpine when available -FROM debian:12-slim -ADD artifacts.tar.gz /install -RUN dpkg -i /install/artifacts/sci_*-1_amd64.deb -RUN rm -rf /install -ENTRYPOINT ["sci"] diff --git a/.sci.sh b/.sci.sh index a358d9a..f9a8444 100755 --- a/.sci.sh +++ b/.sci.sh @@ -2,6 +2,7 @@ set -e echo ">>> checking if required environment is set..." test -n "$DOCKER_TOKEN" +which make echo ">>> compiling..." make @@ -12,7 +13,7 @@ make dist SRC_SHA256=$(sha256sum "sci-${VERSION}.tar.gz" | awk '{ print $1 }') sed "s/SRC_SHA256/${SRC_SHA256}/g" < PKGBUILD.in > PKGBUILD -# arch +# # arch echo ">>> building archbuilder image..." docker build -t archbuilder -f arch-builder.dockerfile . @@ -29,7 +30,7 @@ echo ">>> building debbuilder image..." docker build -t debbuilder -f deb-builder.dockerfile . echo ">>> building .deb in debbuilder docker image..." -docker run --rm -it -v .:/src -e VERSION debbuilder sh -c '\ +docker run --rm -it -v .:/src -e VERSION -e DOCKER_TOKEN debbuilder sh -c '\ cd && \ mkdir -p artifacts && \ cp /src/sci-$VERSION.tar.gz . && \ @@ -46,14 +47,8 @@ docker run --rm -it -v .:/src -e VERSION debbuilder sh -c '\ cp ../*.tar.xz ~/artifacts && \ cp ../*.tar.gz ~/artifacts && \ cd && \ - tar czf /src/artifacts.tar.gz artifacts + curl --user agj:$DOCKER_TOKEN \ + --upload-file sci_$VERSION-1_amd64.deb \ + "https://git.gtz.dk/api/packages/agj/debian/pool/bionic/main/upload" ' - -echo ">>> building sci docker image..." -export OWNER="git.gtz.dk/agj" -docker build -t ${OWNER}/sci:${VERSION} -t ${OWNER}/sci:latest -f .dockerfile . - -echo ">>> pushing latest docker image..." -# TODO: user should be some sci-bot or something, not your account. This will do for now though -docker login git.gtz.dk -u agj -p "$DOCKER_TOKEN" -docker push ${OWNER}/sci:latest +# TODO: push-user should be some sci-bot or something, not your account. This will do for now though diff --git a/Makefile b/Makefile index 4bf0d8d..cabe7ea 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,8 @@ OBJ += out/obj/util.o OBJ += out/obj/pipeline.o OBJ += out/obj/threadlist.o OBJ += out/obj/threadpool.o +OBJ += out/obj/argv_split.o +OBJ += out/obj/which.o out/bin/sci: $(OBJ) | $(BINDIR) $(CC) -o $@ $^ $(CFLAGS) diff --git a/TODO.md b/TODO.md index 2395a32..5414299 100644 --- a/TODO.md +++ b/TODO.md @@ -17,15 +17,17 @@ - [-] ~~alpine~~ later. - [-] ~~docker~~ later. - [ ] Eight things eight, try it out! - maybe even write the python webhook extension. + - [ ] Port this document to gitea issue tracking - [ ] Ninth things ninth, fix bugs, see below - [ ] Tenth things tenth, write manpages, choose license - [ ] Eleventh things Eleventh, polish - [ ] Twelveth things last, release! - - Setup gitea.gtz.dk (will learn you how to set up subdomains (useful for shop.gtz.dk)) + - [x] Setup git.gtz.dk (will learn you how to set up subdomains (useful for shop.gtz.dk)) + - [ ] -1th things -1th, write a blog post about the tool (also set up your blog.gtz.dk) -BOOKMARK: okay. Now it feels like it's getting complicated. I want to run `sci` in a docker container. But that means +Okay. Now it feels like it's getting complicated. I want to run `sci` in a docker container. But that means that the build-threads also run in that docker container - meaning the container should have all the build dependencies -installed and we all know where that rabbithole goes. 9-30YB docker images with about a trillion unique build systems. +installed and we all know where that rabbithole goes. 9-30YiB docker images with about a trillion unique build systems. Let's not do that. The only alternative I can see is that the `sci` service is just not dockerized. The pipeline scripts can easily be dockerized themselves. Just have a `scripts/wget-src-dist-and-sci-sh-dockerized.sh` with `arg1` being the docker image @@ -33,12 +35,24 @@ to use? ```sh #!/bin/sh wget "$SCI_PIPELINE_URL" -docker run --rm -it --mount type=bind,source="$(pwd)"/thefileyouwgot.tar.gz,target=/thefileyouwgot.tar.gz,readonly --entrypoint sh $2 +docker run --rm -it -v .:/src -w /src $@ ``` Or something like that... Perhaps we can figure something out with an inline `ADD`, that also extracts the archive in the container or something. This approach is cleaner IMO. You can also more easily edit the `pipelines.conf` file if you need to. +The aforementioned rabbithole went like this: + - Let's say that `sci` is run inside a docker container. + This would make it very easy to deploy, but: + - Since pipelines are executed in the same environment as `sci`, either: + The `sci` container must be all-encompassing. i.e. it contains every single build system and scriptling language that + could possibly be used by any kind of user or; all pipelines must be run from a docker container themselves, meaning + that the `sci` container must have `dind`-privileges. Either option is suboptimal and will lock users into one way of + using `sci`, which is bad. + - Conclusion: Fuck docker. All environment management is delegated to the user and is not `sci`'s responsibility! + `sci` will always be run on the ci-machine itself, unless a user has provided a custom docker image, which is fine + and doesn't burden the `sci` project. + You were getting the following `pipelines.conf` file to work: ``` scih-dev ssh://git@git.gtz.dk:222/agj/scih.git scih-onpush /etc/sci/scripts/git-clone-and-run-sci-sh.sh @@ -69,7 +83,7 @@ docker is super easy, just make a dockerfile - only concern is the trigger files - [ ] Custom environment variables passed to the pipelines on invokation should be possible. - [ ] Listener threads should be killed and restarted (worker pool should just chug along) when pipeline config file has changed during runtime. Should be disableable with `--no-hot-reload-config` - i.e. on by default. - - [ ] `docker stop` is very slow. I am probably not handling signals properly yet. + - [x] ~~`docker stop` is very slow. I am probably not handling signals properly yet.~~ native docker is abandonned - [x] It seems that `-v 4` is segfaulting when running release builds, maybe the logger just cant find the source file? Nope. I just wrote some bad code (inverted NULL check). diff --git a/include/argv_split.h b/include/argv_split.h new file mode 100644 index 0000000..f8de895 --- /dev/null +++ b/include/argv_split.h @@ -0,0 +1,24 @@ +/** + * sci - a simple ci system + Copyright (C) 2024 Asger Gitz-Johansen + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ +#ifndef SCI_ARGV_SPLIT_H +#define SCI_ARGV_SPLIT_H + +char** create_argv_shell(const char* str, int* argc_out); +void argv_free(char** argv); + +#endif diff --git a/include/which.h b/include/which.h new file mode 100644 index 0000000..c76ac45 --- /dev/null +++ b/include/which.h @@ -0,0 +1,23 @@ +/** + * sci - a simple ci system + Copyright (C) 2024 Asger Gitz-Johansen + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ +#ifndef SCI_WHICH_H +#define SCI_WHICH_H + +int which(const char* program_name, char* out_full_program, int max_path); + +#endif diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..c381a22 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,3 @@ +# sci default scripts +This directory contains some default scripts that may or may not be useful to you. +Most of the scripts are fairly simple, but should be installed as part of the sci installation process. diff --git a/scripts/git-clone-and-run-sci-sh.sh b/scripts/git-clone-and-run-sci-sh.sh deleted file mode 100644 index 9d50675..0000000 --- a/scripts/git-clone-and-run-sci-sh.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -set -e -echo ">>> cloning..." -git clone $SCI_PIPELINE_URL $SCI_PIPELINE_NAME -cd $SCI_PIPELINE_NAME - -echo ">>> running .sci.sh..." -time sh .sci.sh diff --git a/scripts/git-clone-and-sci.sh b/scripts/git-clone-and-sci.sh new file mode 100755 index 0000000..ab08dcc --- /dev/null +++ b/scripts/git-clone-and-sci.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -ex # print all that we're doing (no need for echo's) +tmpdir=$(mktemp -d) +git clone --depth=1 --recurse-submodules --shallow-submodules -b $1 "$SCI_PIPELINE_URL" "$tmpdir" +shift +cd "$tmpdir" +sh .sci.sh +cd - diff --git a/scripts/wget-and-sci.sh b/scripts/wget-and-sci.sh new file mode 100755 index 0000000..4ee671f --- /dev/null +++ b/scripts/wget-and-sci.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# NOTE: This script assumes that the url is a .tar.gz file. +# TODO: check if $# is >= 1 and give a warning that the extract dir should be provided. +set -ex # print all that we're doing (no need for echo's) +tmpdir=$(mktemp -d) +wget "$SCI_PIPELINE_URL" -P "$tmpdir" +cd "$tmpdir" +tar xf *.tar.gz +cd $1 +sh .sci.sh +cd - +rm -rf "$tmpdir" diff --git a/src/argv_split.c b/src/argv_split.c new file mode 100644 index 0000000..66751a3 --- /dev/null +++ b/src/argv_split.c @@ -0,0 +1,81 @@ +/** + * sci - a simple ci system + Copyright (C) 2024 Asger Gitz-Johansen + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ +#include "argv_split.h" +#include +#include +#include +#include + +static const char* skip_arg(const char* cp) { + while(*cp && !isspace(*cp)) + cp++; + return cp; +} + +static char* skip_spaces(const char* str) { + while(isspace(*str)) + str++; + return(char*)str; +} + +static int count_argc(const char* str) { + int count = 0; + while(*str) { + str = skip_spaces(str); + if(!*str) + continue; + count++; + str = skip_arg(str); + } + return count; +} + +void argv_free(char** argv) { + for(char** p = argv; *p; p++) { + free(*p); + *p = NULL; + } + free(argv); +} + +char** create_argv_shell(const char* str, int* argc_out) { + int argc = count_argc(str); + char** result = calloc(argc+1, sizeof(*result)); + if(result == NULL) + return result; + if(argc_out) + *argc_out = argc+1; + char** argvp = result; + while(*str) { + str = skip_spaces(str); + if(!*str) + continue; + const char* p = str; + str = skip_arg(str); + char* t = strndup(p, str-p); + if(t == NULL) { + perror("strndup"); + argv_free(result); + return NULL; + } + *argvp++ = t; + } + *argvp = NULL; + return result; +} +// sci-release https://git.gtz.dk/agj/sci/archive/main.tar.gz manual "scripts/wget-and-sci.sh sci" diff --git a/src/main.c b/src/main.c index a68048b..1833fa0 100644 --- a/src/main.c +++ b/src/main.c @@ -15,16 +15,20 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ +#include "argv_split.h" #include "cli.h" #include "log.h" #include "notify.h" #include "pipeline.h" #include "threadpool.h" #include "util.h" +#include "which.h" #include +#include #include #include #include +#include #include #include @@ -62,8 +66,10 @@ void executor(void* data) { // Create logfile path optional_int fd = open_logfile(pipeline_id); - if(!fd.has_value) + if(!fd.has_value) { + log_error("could not open log file - not starting pipeline"); return; + } // spawn the process pid_t pid; @@ -72,24 +78,35 @@ void executor(void* data) { posix_spawn_file_actions_adddup2(&actions, fd.value, STDOUT_FILENO); posix_spawn_file_actions_adddup2(&actions, fd.value, STDERR_FILENO); const pipeline_event* const e = data; + char* path = join("PATH=", getenv("PATH")); char* name = join("SCI_PIPELINE_NAME=", e->name); char* url = join("SCI_PIPELINE_URL=", e->url); char* trigger = join("SCI_PIPELINE_TRIGGER=", e->trigger); char* id = join("SCI_PIPELINE_ID=", pipeline_id); - char* envp[] = { name, url, trigger, id, NULL }; - char* argv[] = { "/bin/sh", "-c", e->command, NULL }; - if(posix_spawn(&pid, "/bin/sh", &actions, NULL, argv, envp) != 0) { + char* envp[] = { path, name, url, trigger, id, NULL }; + int argc; + char** argv = create_argv_shell(e->command, &argc); + log_trace("executing pipeline %s with argv:", e->name); + for(int i = 0; i < argc; i++) + log_trace(" \"%s\"", argv[i]); + char arg0[PATH_MAX]; + if(which(argv[0], arg0, PATH_MAX) == -1) + goto end; + if(posix_spawn(&pid, arg0, &actions, NULL, argv, envp) != 0) { perror("posix_spawn"); - goto end; // I know. The raptors have picked up the scent. I'll just have to mask it with more poopy code. + goto end; // I know. The raptors have picked up the scent. I'll just have to mask it with more stinky code. } - log_trace("{%s} (%s) spawned", pipeline_id, e->name); + log_info("{%s} (%s) spawned", pipeline_id, e->name); // Wait for process to complete int status; waitpid(pid, &status, 0); - if(WIFEXITED(status)) - log_trace("{%s} (%s) exited with status %d", pipeline_id, e->name, WEXITSTATUS(status)); + log_info("{%s} (%s) exited with status %d", pipeline_id, e->name, status); + char buf[32]; + sprintf(buf, "exited with status %d", status); + write(fd.value, buf, strnlen(buf, 32)); end: + argv_free(argv); close(fd.value); free(pipeline_id); free(name); diff --git a/src/pipeline.c b/src/pipeline.c index cc2feef..4e3f131 100644 --- a/src/pipeline.c +++ b/src/pipeline.c @@ -40,7 +40,11 @@ optional_pipeline_conf pipeline_create(const char* config_line) { break; off = pmatch[0].rm_so + (cursor - config_line); len = pmatch[0].rm_eo - pmatch[0].rm_so; - opts[i] = strndup(config_line + off, len); + // Cut off the "-s if it is string-enclosed + if(config_line[off] == '"' && config_line[off+len-1] == '"') + opts[i] = strndup(config_line + off+1, len-2); + else + opts[i] = strndup(config_line + off, len); cursor += pmatch[0].rm_eo; } if(i != 4) { diff --git a/src/which.c b/src/which.c new file mode 100644 index 0000000..7e7c8b4 --- /dev/null +++ b/src/which.c @@ -0,0 +1,54 @@ +/** + * sci - a simple ci system + Copyright (C) 2024 Asger Gitz-Johansen + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ +#include "log.h" +#include +#include +#include +#include +#include +#include + +int which(const char* program_name, char* out_full_program, int max_path) { + assert(out_full_program); + assert(max_path > 0); + // sanity check - maybe program_name is actually a full-path to begin with + if(access(program_name, X_OK) == 0) { + snprintf(out_full_program, max_path, "%s", program_name); + return 0; + } + char* path = getenv("PATH"); + if (path == NULL) { + log_error("PATH environment variable not found."); + return -1; + } + char* path_cpy = strdup(path); + char* dir = strtok(path_cpy, ":"); + char full_path[PATH_MAX]; + while(dir != NULL) { + snprintf(full_path, sizeof(full_path), "%s/%s", dir, program_name); + if(access(full_path, X_OK) == 0) { + snprintf(out_full_program, max_path, "%s", full_path); + free(path_cpy); + return 0; + } + dir = strtok(NULL, ":"); + } + log_error("'%s' not found in PATH", program_name); + free(path_cpy); + return -1; +} -- 2.52.0 From e44280077995c506a1de382da2023043f6ab7043 Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Sun, 25 Aug 2024 15:04:04 +0200 Subject: [PATCH 02/18] fix: move the new utils into utils.h/c --- Makefile | 2 - include/argv_split.h | 24 ------------ include/util.h | 7 ++++ include/which.h | 23 ----------- src/argv_split.c | 81 --------------------------------------- src/main.c | 4 +- src/util.c | 91 +++++++++++++++++++++++++++++++++++++++++++- src/which.c | 54 -------------------------- 8 files changed, 98 insertions(+), 188 deletions(-) delete mode 100644 include/argv_split.h delete mode 100644 include/which.h delete mode 100644 src/argv_split.c delete mode 100644 src/which.c diff --git a/Makefile b/Makefile index cabe7ea..4bf0d8d 100644 --- a/Makefile +++ b/Makefile @@ -37,8 +37,6 @@ OBJ += out/obj/util.o OBJ += out/obj/pipeline.o OBJ += out/obj/threadlist.o OBJ += out/obj/threadpool.o -OBJ += out/obj/argv_split.o -OBJ += out/obj/which.o out/bin/sci: $(OBJ) | $(BINDIR) $(CC) -o $@ $^ $(CFLAGS) diff --git a/include/argv_split.h b/include/argv_split.h deleted file mode 100644 index f8de895..0000000 --- a/include/argv_split.h +++ /dev/null @@ -1,24 +0,0 @@ -/** - * sci - a simple ci system - Copyright (C) 2024 Asger Gitz-Johansen - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - */ -#ifndef SCI_ARGV_SPLIT_H -#define SCI_ARGV_SPLIT_H - -char** create_argv_shell(const char* str, int* argc_out); -void argv_free(char** argv); - -#endif diff --git a/include/util.h b/include/util.h index 57095ab..d85d8e2 100644 --- a/include/util.h +++ b/include/util.h @@ -41,4 +41,11 @@ void per_line(const char* file, line_handler handler); char* join(const char* a, const char* b); +const char* skip_arg(const char* cp); +char* skip_spaces(const char* str); +int count_argc(const char* str); +char** argv_split(const char* str, int* argc_out); +void argv_free(char** argv); +int which(const char* program_name, char* out_full_program, int max_path); + #endif diff --git a/include/which.h b/include/which.h deleted file mode 100644 index c76ac45..0000000 --- a/include/which.h +++ /dev/null @@ -1,23 +0,0 @@ -/** - * sci - a simple ci system - Copyright (C) 2024 Asger Gitz-Johansen - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - */ -#ifndef SCI_WHICH_H -#define SCI_WHICH_H - -int which(const char* program_name, char* out_full_program, int max_path); - -#endif diff --git a/src/argv_split.c b/src/argv_split.c deleted file mode 100644 index 66751a3..0000000 --- a/src/argv_split.c +++ /dev/null @@ -1,81 +0,0 @@ -/** - * sci - a simple ci system - Copyright (C) 2024 Asger Gitz-Johansen - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - */ -#include "argv_split.h" -#include -#include -#include -#include - -static const char* skip_arg(const char* cp) { - while(*cp && !isspace(*cp)) - cp++; - return cp; -} - -static char* skip_spaces(const char* str) { - while(isspace(*str)) - str++; - return(char*)str; -} - -static int count_argc(const char* str) { - int count = 0; - while(*str) { - str = skip_spaces(str); - if(!*str) - continue; - count++; - str = skip_arg(str); - } - return count; -} - -void argv_free(char** argv) { - for(char** p = argv; *p; p++) { - free(*p); - *p = NULL; - } - free(argv); -} - -char** create_argv_shell(const char* str, int* argc_out) { - int argc = count_argc(str); - char** result = calloc(argc+1, sizeof(*result)); - if(result == NULL) - return result; - if(argc_out) - *argc_out = argc+1; - char** argvp = result; - while(*str) { - str = skip_spaces(str); - if(!*str) - continue; - const char* p = str; - str = skip_arg(str); - char* t = strndup(p, str-p); - if(t == NULL) { - perror("strndup"); - argv_free(result); - return NULL; - } - *argvp++ = t; - } - *argvp = NULL; - return result; -} -// sci-release https://git.gtz.dk/agj/sci/archive/main.tar.gz manual "scripts/wget-and-sci.sh sci" diff --git a/src/main.c b/src/main.c index 1833fa0..aa8fd3f 100644 --- a/src/main.c +++ b/src/main.c @@ -15,14 +15,12 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -#include "argv_split.h" #include "cli.h" #include "log.h" #include "notify.h" #include "pipeline.h" #include "threadpool.h" #include "util.h" -#include "which.h" #include #include #include @@ -85,7 +83,7 @@ void executor(void* data) { char* id = join("SCI_PIPELINE_ID=", pipeline_id); char* envp[] = { path, name, url, trigger, id, NULL }; int argc; - char** argv = create_argv_shell(e->command, &argc); + char** argv = argv_split(e->command, &argc); log_trace("executing pipeline %s with argv:", e->name); for(int i = 0; i < argc; i++) log_trace(" \"%s\"", argv[i]); diff --git a/src/util.c b/src/util.c index e49ac71..bc08701 100644 --- a/src/util.c +++ b/src/util.c @@ -15,9 +15,10 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -#include "util.h" #include "log.h" +#include "util.h" #include +#include #include char* trim(const char* const str) { @@ -61,3 +62,91 @@ char* join(const char* a, const char* b) { sprintf(result, "%s%s", a, b); return result; } + +const char* skip_arg(const char* cp) { + while(*cp && !isspace(*cp)) + cp++; + return cp; +} + +char* skip_spaces(const char* str) { + while(isspace(*str)) + str++; + return(char*)str; +} + +int count_argc(const char* str) { + int count = 0; + while(*str) { + str = skip_spaces(str); + if(!*str) + continue; + count++; + str = skip_arg(str); + } + return count; +} + +void argv_free(char** argv) { + for(char** p = argv; *p; p++) { + free(*p); + *p = NULL; + } + free(argv); +} + +char** argv_split(const char* str, int* argc_out) { + int argc = count_argc(str); + char** result = calloc(argc+1, sizeof(*result)); + if(result == NULL) + return result; + if(argc_out) + *argc_out = argc+1; + char** argvp = result; + while(*str) { + str = skip_spaces(str); + if(!*str) + continue; + const char* p = str; + str = skip_arg(str); + char* t = strndup(p, str-p); + if(t == NULL) { + perror("strndup"); + argv_free(result); + return NULL; + } + *argvp++ = t; + } + *argvp = NULL; + return result; +} + +int which(const char* program_name, char* out_full_program, int max_path) { + assert(out_full_program); + assert(max_path > 0); + // sanity check - maybe program_name is actually a full-path to begin with + if(access(program_name, X_OK) == 0) { + snprintf(out_full_program, max_path, "%s", program_name); + return 0; + } + char* path = getenv("PATH"); + if (path == NULL) { + log_error("PATH environment variable not found."); + return -1; + } + char* path_cpy = strdup(path); + char* dir = strtok(path_cpy, ":"); + char full_path[PATH_MAX]; + while(dir != NULL) { + snprintf(full_path, sizeof(full_path), "%s/%s", dir, program_name); + if(access(full_path, X_OK) == 0) { + snprintf(out_full_program, max_path, "%s", full_path); + free(path_cpy); + return 0; + } + dir = strtok(NULL, ":"); + } + log_error("'%s' not found in PATH", program_name); + free(path_cpy); + return -1; +} diff --git a/src/which.c b/src/which.c deleted file mode 100644 index 7e7c8b4..0000000 --- a/src/which.c +++ /dev/null @@ -1,54 +0,0 @@ -/** - * sci - a simple ci system - Copyright (C) 2024 Asger Gitz-Johansen - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - */ -#include "log.h" -#include -#include -#include -#include -#include -#include - -int which(const char* program_name, char* out_full_program, int max_path) { - assert(out_full_program); - assert(max_path > 0); - // sanity check - maybe program_name is actually a full-path to begin with - if(access(program_name, X_OK) == 0) { - snprintf(out_full_program, max_path, "%s", program_name); - return 0; - } - char* path = getenv("PATH"); - if (path == NULL) { - log_error("PATH environment variable not found."); - return -1; - } - char* path_cpy = strdup(path); - char* dir = strtok(path_cpy, ":"); - char full_path[PATH_MAX]; - while(dir != NULL) { - snprintf(full_path, sizeof(full_path), "%s/%s", dir, program_name); - if(access(full_path, X_OK) == 0) { - snprintf(out_full_program, max_path, "%s", full_path); - free(path_cpy); - return 0; - } - dir = strtok(NULL, ":"); - } - log_error("'%s' not found in PATH", program_name); - free(path_cpy); - return -1; -} -- 2.52.0 From 05701d9d85493961d3bc08fa355205ea630c6b18 Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Sun, 25 Aug 2024 15:52:43 +0200 Subject: [PATCH 03/18] wip: custom environment variable passing You should be able to tell your sci deployment which env vars should be passed to the pipelines with -e ENV1 -e ENV2 and so on --- Makefile | 5 +- TODO.md | 2 + include/cli.h | 1 + include/optional.h | 2 + include/strlist.h | 51 +++++++++++++++++ include/threadlist.h | 5 +- scripts/wget-and-sci.sh | 1 + src/cli.c | 38 ++++++++++--- src/sci.1 | 1 - src/strlist.c | 62 +++++++++++++++++++++ src/util.c | 120 ++++++++++++++++++++-------------------- 11 files changed, 215 insertions(+), 73 deletions(-) create mode 100644 include/strlist.h create mode 100644 src/strlist.c diff --git a/Makefile b/Makefile index 4bf0d8d..a1eb224 100644 --- a/Makefile +++ b/Makefile @@ -29,14 +29,15 @@ all: out/bin/sci out/obj/%.o: src/%.c | $(OBJDIR) $(CC) -c $? $(CFLAGS) -o $@ -OBJ += out/obj/main.o OBJ += out/obj/cli.o OBJ += out/obj/log.o +OBJ += out/obj/main.o OBJ += out/obj/notify.o -OBJ += out/obj/util.o OBJ += out/obj/pipeline.o +OBJ += out/obj/strlist.o OBJ += out/obj/threadlist.o OBJ += out/obj/threadpool.o +OBJ += out/obj/util.o out/bin/sci: $(OBJ) | $(BINDIR) $(CC) -o $@ $^ $(CFLAGS) diff --git a/TODO.md b/TODO.md index 5414299..d4a5a66 100644 --- a/TODO.md +++ b/TODO.md @@ -18,6 +18,8 @@ - [-] ~~docker~~ later. - [ ] Eight things eight, try it out! - maybe even write the python webhook extension. - [ ] Port this document to gitea issue tracking + - [x] enable PATH-able programs and argv in the command section + - [ ] custom environment variable passing. Something like `-e MY_TOKEN` ala docker-style - [ ] Ninth things ninth, fix bugs, see below - [ ] Tenth things tenth, write manpages, choose license - [ ] Eleventh things Eleventh, polish diff --git a/include/cli.h b/include/cli.h index 95cf127..f3f511c 100644 --- a/include/cli.h +++ b/include/cli.h @@ -29,6 +29,7 @@ typedef struct { bool use_colors; optional_str log_file; optional_str pipeline_log_dir; + optional_strlist environment_vars; } cli_options; // Construct a new cli_options struct instance. diff --git a/include/optional.h b/include/optional.h index 77ebfc1..b1b2e1e 100644 --- a/include/optional.h +++ b/include/optional.h @@ -17,6 +17,7 @@ */ #ifndef SCI_OPTIONAL_H #define SCI_OPTIONAL_H +#include "strlist.h" #include #define optional_type(type) struct { bool has_value; type value; } @@ -24,5 +25,6 @@ typedef optional_type(int) optional_int; typedef optional_type(float) optional_float; typedef optional_type(char*) optional_str; typedef optional_type(const char*) optional_cstr; +typedef optional_type(strlist_node*) optional_strlist; #endif diff --git a/include/strlist.h b/include/strlist.h new file mode 100644 index 0000000..2298ecd --- /dev/null +++ b/include/strlist.h @@ -0,0 +1,51 @@ +/** + * sci - a simple ci system + Copyright (C) 2024 Asger Gitz-Johansen + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ +#ifndef SCI_STRLIST_H +#define SCI_STRLIST_H + +// doubly linked list +typedef struct strlist_node { + char* str; + struct strlist_node* previous; + struct strlist_node* next; +} strlist_node; + +// Create a new root node. +// This function is not threadsafe. +strlist_node* create_strlist_node(char* str); + +// Add a new string to the string list. +// This function is not threadsafe. +strlist_node* add_str(char* str, strlist_node* root); + +// add a new string list node to the list. +// This function is not threadsafe. +strlist_node* add_str_node(strlist_node* str_node, strlist_node* root); + +// Remove a string list node from the list. +// This will free the str and stitch the "previous" and "next" ptrs. +// This function is not threadsafe. +void remove_strlist_node(strlist_node* node); + +// Completely clear the list. +// The list is completely invalid after this call and should be discarded. +// root itself will not be free'd by this function, but all content will be. +// This function is not threadsafe. +void clear_strlist(strlist_node* root); + +#endif diff --git a/include/threadlist.h b/include/threadlist.h index d7fb249..ce62459 100644 --- a/include/threadlist.h +++ b/include/threadlist.h @@ -42,9 +42,8 @@ void remove_thread_node(pthread_list_node* node); // Completely clear the thread list. // This will call pthread_join on all nodes. // The list is completely invalid after this call and should be discarded. -// Note: -// - `root` has already been free'd. -// - this function is not thread-safe. +// Even root itself will be free'd by this function so it should be discarded as well. +// This function is not thread-safe. void clear_thread_list(pthread_list_node* root); #endif diff --git a/scripts/wget-and-sci.sh b/scripts/wget-and-sci.sh index 4ee671f..1b79fc3 100755 --- a/scripts/wget-and-sci.sh +++ b/scripts/wget-and-sci.sh @@ -2,6 +2,7 @@ # NOTE: This script assumes that the url is a .tar.gz file. # TODO: check if $# is >= 1 and give a warning that the extract dir should be provided. set -ex # print all that we're doing (no need for echo's) +env tmpdir=$(mktemp -d) wget "$SCI_PIPELINE_URL" -P "$tmpdir" cd "$tmpdir" diff --git a/src/cli.c b/src/cli.c index c8b8aa3..ade9c6b 100644 --- a/src/cli.c +++ b/src/cli.c @@ -15,11 +15,12 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ +#include "cli.h" +#include "log.h" +#include #include #include #include -#include -#include "cli.h" cli_options new_options() { cli_options result; @@ -50,6 +51,19 @@ cli_options new_options() { char* pipeline_log_dir = getenv("SCI_PIPELINE_LOG_DIR"); result.pipeline_log_dir.has_value = pipeline_log_dir != NULL; result.pipeline_log_dir.value = pipeline_log_dir; + + char* environment_vars = getenv("SCI_PIPELINE_ENV_VARS"); + if(environment_vars == NULL) { + result.environment_vars.has_value = false; + result.environment_vars.value = NULL; + } else { + char* tok = strtok(environment_vars, ";"); + result.environment_vars.has_value = true; + result.environment_vars.value = create_strlist_node(tok); + tok = strtok(NULL, ";"); + while(tok != NULL) + add_str(tok, result.environment_vars.value); + } return result; } @@ -60,24 +74,27 @@ void destroy_options(cli_options v) { free(v.log_file.value); if(v.pipeline_log_dir.has_value) free(v.pipeline_log_dir.value); + if(v.environment_vars.has_value) + clear_strlist(v.environment_vars.value); } // . -\" TODO: decide on license .SH COPYRIGHT Copyright (C) 2024 Asger Gitz-Johansen diff --git a/src/strlist.c b/src/strlist.c new file mode 100644 index 0000000..6fe70e3 --- /dev/null +++ b/src/strlist.c @@ -0,0 +1,62 @@ +#include "strlist.h" +#include +#include + +#define MAX_STRLEN 512 + +strlist_node* create_strlist_node(char* str) { + strlist_node* new = malloc(sizeof(strlist_node)); + new->previous = NULL; + new->next = NULL; + if(str) + new->str = strndup(str, MAX_STRLEN); + else + new->str = NULL; + return new; +} + +strlist_node* add_str(char* str, strlist_node* root) { + strlist_node* cursor = root; + while(cursor->next != NULL) + cursor = cursor->next; + strlist_node* new = malloc(sizeof(strlist_node)); + new->previous = cursor; + new->next = NULL; + new->str = strndup(str, MAX_STRLEN); + cursor->next = new; + return new; +} + +strlist_node* add_str_node(strlist_node* root, strlist_node* node) { + strlist_node* cursor = root; + while(cursor->next != NULL) + cursor = cursor->next; + node->previous = cursor; + node->next = NULL; + cursor->next = node; + return node; +} + +void remove_strlist_node(strlist_node* node) { + strlist_node* prev = node->previous; + strlist_node* next = node->next; + if(node->str) + free(node->str); + node->str = NULL; + free(node); + if(prev != NULL) + prev->next = next; + if(next != NULL) + next->previous = prev; +} + +void clear_strlist(strlist_node* root) { + strlist_node* cursor = root; + while(cursor != NULL) { + cursor = cursor->next; + if(cursor->str) + free(cursor->str); + cursor->str = NULL; + free(cursor->previous); + } +} diff --git a/src/util.c b/src/util.c index bc08701..1f8973a 100644 --- a/src/util.c +++ b/src/util.c @@ -22,45 +22,45 @@ #include char* trim(const char* const str) { - char* begin = strdup(str); - char* end; - while(isspace((unsigned char)*begin)) - begin++; - if(*begin == 0) - return begin; - end = begin + strlen(begin) - 1; - while(end > begin && isspace((unsigned char)*end)) - end--; - *(end + 1) = '\0'; - return begin; + char* begin = strdup(str); + char* end; + while(isspace((unsigned char)*begin)) + begin++; + if(*begin == 0) + return begin; + end = begin + strlen(begin) - 1; + while(end > begin && isspace((unsigned char)*end)) + end--; + *(end + 1) = '\0'; + return begin; } void per_line(const char* file, line_handler handler) { - FILE* stream; - char* line = NULL; - size_t len = 0; - ssize_t nread; - log_trace("reading file %s", file); - stream = fopen(file, "r"); - if(stream == NULL) { - perror("fopen"); - return; - } - while((nread = getline(&line, &len, stream)) != -1) { - char* line_trimmed = trim(line); - handler(line_trimmed); - free(line_trimmed); - } - free(line); - fclose(stream); + FILE* stream; + char* line = NULL; + size_t len = 0; + ssize_t nread; + log_trace("reading file %s", file); + stream = fopen(file, "r"); + if(stream == NULL) { + perror("fopen"); + return; + } + while((nread = getline(&line, &len, stream)) != -1) { + char* line_trimmed = trim(line); + handler(line_trimmed); + free(line_trimmed); + } + free(line); + fclose(stream); } char* join(const char* a, const char* b) { - size_t alen = strlen(a); - size_t blen = strlen(b); - char* result = malloc(alen + blen + 1); - sprintf(result, "%s%s", a, b); - return result; + size_t alen = strlen(a); + size_t blen = strlen(b); + char* result = malloc(alen + blen + 1); + sprintf(result, "%s%s", a, b); + return result; } const char* skip_arg(const char* cp) { @@ -122,31 +122,31 @@ char** argv_split(const char* str, int* argc_out) { } int which(const char* program_name, char* out_full_program, int max_path) { - assert(out_full_program); - assert(max_path > 0); - // sanity check - maybe program_name is actually a full-path to begin with - if(access(program_name, X_OK) == 0) { - snprintf(out_full_program, max_path, "%s", program_name); - return 0; - } - char* path = getenv("PATH"); - if (path == NULL) { - log_error("PATH environment variable not found."); - return -1; - } - char* path_cpy = strdup(path); - char* dir = strtok(path_cpy, ":"); - char full_path[PATH_MAX]; - while(dir != NULL) { - snprintf(full_path, sizeof(full_path), "%s/%s", dir, program_name); - if(access(full_path, X_OK) == 0) { - snprintf(out_full_program, max_path, "%s", full_path); - free(path_cpy); - return 0; - } - dir = strtok(NULL, ":"); - } - log_error("'%s' not found in PATH", program_name); - free(path_cpy); - return -1; + assert(out_full_program); + assert(max_path > 0); + // sanity check - maybe program_name is actually a full-path to begin with + if(access(program_name, X_OK) == 0) { + snprintf(out_full_program, max_path, "%s", program_name); + return 0; + } + char* path = getenv("PATH"); + if (path == NULL) { + log_error("PATH environment variable not found."); + return -1; + } + char* path_cpy = strdup(path); + char* dir = strtok(path_cpy, ":"); + char full_path[PATH_MAX]; + while(dir != NULL) { + snprintf(full_path, sizeof(full_path), "%s/%s", dir, program_name); + if(access(full_path, X_OK) == 0) { + snprintf(out_full_program, max_path, "%s", full_path); + free(path_cpy); + return 0; + } + dir = strtok(NULL, ":"); + } + log_error("'%s' not found in PATH", program_name); + free(path_cpy); + return -1; } -- 2.52.0 From 0f1aa982f4414c80556ba4b53784c94881b48082 Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Mon, 26 Aug 2024 18:40:23 +0200 Subject: [PATCH 04/18] fixup! refactor: move executor function into it's own file --- Makefile | 1 + include/executor.h | 7 ++++ src/cli.c | 1 - src/executor.c | 102 +++++++++++++++++++++++++++++++++++++++++++++ src/main.c | 94 ++--------------------------------------- 5 files changed, 113 insertions(+), 92 deletions(-) create mode 100644 include/executor.h create mode 100644 src/executor.c diff --git a/Makefile b/Makefile index a1eb224..83f0962 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ out/obj/%.o: src/%.c | $(OBJDIR) $(CC) -c $? $(CFLAGS) -o $@ OBJ += out/obj/cli.o +OBJ += out/obj/executor.o OBJ += out/obj/log.o OBJ += out/obj/main.o OBJ += out/obj/notify.o diff --git a/include/executor.h b/include/executor.h new file mode 100644 index 0000000..7c42747 --- /dev/null +++ b/include/executor.h @@ -0,0 +1,7 @@ +#ifndef SCI_EXECUTOR_H +#define SCI_EXECUTOR_H + +void executor(void* pipeline_event); +void set_logdir(const char* logdir); + +#endif diff --git a/src/cli.c b/src/cli.c index ade9c6b..317b936 100644 --- a/src/cli.c +++ b/src/cli.c @@ -16,7 +16,6 @@ along with this program. If not, see . */ #include "cli.h" -#include "log.h" #include #include #include diff --git a/src/executor.c b/src/executor.c new file mode 100644 index 0000000..08fe358 --- /dev/null +++ b/src/executor.c @@ -0,0 +1,102 @@ +#include "executor.h" +#include "log.h" +#include "optional.h" +#include "pipeline.h" +#include "util.h" +#include +#include +#include +#include +#include +#include +#include +#include + +const char* log_dir = "./"; // NOTE: must end with a / + +void set_logdir(const char* logdir) { + log_dir = logdir; + struct stat st = {0}; + if(stat(log_dir, &st) == -1) + mkdir(log_dir, 0700); +} + +char* create_pipeline_id() { + uuid_t uuid; + uuid_generate(uuid); + char* pipeline_id = malloc(32); + uuid_unparse_lower(uuid, pipeline_id); + return pipeline_id; +} + +optional_int open_logfile(const char* const pipeline_id) { + optional_int result; + result.has_value = false; + result.value = 0; + char* log_file = join(pipeline_id, ".log"); + char* log_filepath = join(log_dir, log_file); + int fd = open(log_filepath, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd != -1) { + result.has_value = true; + result.value = fd; + } else + perror("open"); + free(log_file); + free(log_filepath); + return result; +} + +void executor(void* data) { + // Create pipeline id + char* pipeline_id = create_pipeline_id(); + + // Create logfile path + optional_int fd = open_logfile(pipeline_id); + if(!fd.has_value) { + log_error("could not open log file - not starting pipeline"); + return; + } + + // spawn the process + pid_t pid; + posix_spawn_file_actions_t actions; + posix_spawn_file_actions_init(&actions); + posix_spawn_file_actions_adddup2(&actions, fd.value, STDOUT_FILENO); + posix_spawn_file_actions_adddup2(&actions, fd.value, STDERR_FILENO); + const pipeline_event* const e = data; + char* path = join("PATH=", getenv("PATH")); + char* name = join("SCI_PIPELINE_NAME=", e->name); + char* url = join("SCI_PIPELINE_URL=", e->url); + char* trigger = join("SCI_PIPELINE_TRIGGER=", e->trigger); + char* id = join("SCI_PIPELINE_ID=", pipeline_id); + char* envp[] = { path, name, url, trigger, id, NULL }; + int argc; + char** argv = argv_split(e->command, &argc); + log_trace("executing pipeline %s with argv:", e->name); + for(int i = 0; i < argc; i++) + log_trace(" \"%s\"", argv[i]); + char arg0[PATH_MAX]; + if(which(argv[0], arg0, PATH_MAX) == -1) + goto end; + if(posix_spawn(&pid, arg0, &actions, NULL, argv, envp) != 0) { + perror("posix_spawn"); + goto end; // I know. The raptors have picked up the scent. I'll just have to mask it with more stinky code. + } + log_info("{%s} (%s) spawned", pipeline_id, e->name); + + // Wait for process to complete + int status; + waitpid(pid, &status, 0); + log_info("{%s} (%s) exited with status %d", pipeline_id, e->name, status); + char buf[32]; + sprintf(buf, "exited with status %d", status); + write(fd.value, buf, strnlen(buf, 32)); +end: + argv_free(argv); + close(fd.value); + free(pipeline_id); + free(name); + free(url); + free(trigger); + free(id); +} diff --git a/src/main.c b/src/main.c index aa8fd3f..b726a2f 100644 --- a/src/main.c +++ b/src/main.c @@ -16,102 +16,17 @@ along with this program. If not, see . */ #include "cli.h" +#include "executor.h" #include "log.h" #include "notify.h" #include "pipeline.h" #include "threadpool.h" #include "util.h" -#include -#include -#include +#include #include #include -#include -#include -#include threadpool* pool = NULL; -char* log_dir = "./"; // NOTE: must end with a / - -char* create_pipeline_id() { - uuid_t uuid; - uuid_generate(uuid); - char* pipeline_id = malloc(32); - uuid_unparse_lower(uuid, pipeline_id); - return pipeline_id; -} - -optional_int open_logfile(const char* const pipeline_id) { - optional_int result; - result.has_value = false; - result.value = 0; - char* log_file = join(pipeline_id, ".log"); - char* log_filepath = join(log_dir, log_file); - int fd = open(log_filepath, O_WRONLY | O_CREAT | O_TRUNC, 0644); - if (fd != -1) { - result.has_value = true; - result.value = fd; - } else - perror("open"); - free(log_file); - free(log_filepath); - return result; -} - -void executor(void* data) { - // Create pipeline id - char* pipeline_id = create_pipeline_id(); - - // Create logfile path - optional_int fd = open_logfile(pipeline_id); - if(!fd.has_value) { - log_error("could not open log file - not starting pipeline"); - return; - } - - // spawn the process - pid_t pid; - posix_spawn_file_actions_t actions; - posix_spawn_file_actions_init(&actions); - posix_spawn_file_actions_adddup2(&actions, fd.value, STDOUT_FILENO); - posix_spawn_file_actions_adddup2(&actions, fd.value, STDERR_FILENO); - const pipeline_event* const e = data; - char* path = join("PATH=", getenv("PATH")); - char* name = join("SCI_PIPELINE_NAME=", e->name); - char* url = join("SCI_PIPELINE_URL=", e->url); - char* trigger = join("SCI_PIPELINE_TRIGGER=", e->trigger); - char* id = join("SCI_PIPELINE_ID=", pipeline_id); - char* envp[] = { path, name, url, trigger, id, NULL }; - int argc; - char** argv = argv_split(e->command, &argc); - log_trace("executing pipeline %s with argv:", e->name); - for(int i = 0; i < argc; i++) - log_trace(" \"%s\"", argv[i]); - char arg0[PATH_MAX]; - if(which(argv[0], arg0, PATH_MAX) == -1) - goto end; - if(posix_spawn(&pid, arg0, &actions, NULL, argv, envp) != 0) { - perror("posix_spawn"); - goto end; // I know. The raptors have picked up the scent. I'll just have to mask it with more stinky code. - } - log_info("{%s} (%s) spawned", pipeline_id, e->name); - - // Wait for process to complete - int status; - waitpid(pid, &status, 0); - log_info("{%s} (%s) exited with status %d", pipeline_id, e->name, status); - char buf[32]; - sprintf(buf, "exited with status %d", status); - write(fd.value, buf, strnlen(buf, 32)); -end: - argv_free(argv); - close(fd.value); - free(pipeline_id); - free(name); - free(url); - free(trigger); - free(id); -} void on_event(pipeline_event* const e) { if(!threadpool_add_work(pool, executor, (void*)e)) @@ -185,12 +100,9 @@ int main(int argc, char** argv) { } if(args.pipeline_log_dir.has_value) - log_dir = args.pipeline_log_dir.value; + set_logdir(args.pipeline_log_dir.value); struct stat st = {0}; - if(stat(log_dir, &st) == -1) - mkdir(log_dir, 0700); - if(stat("/tmp/sci", &st) == -1) mkdir("/tmp/sci", 0700); -- 2.52.0 From cfffa4342820c39a50dfe59a5bd989e50a51639d Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Mon, 26 Aug 2024 19:35:05 +0200 Subject: [PATCH 05/18] feat: introduce sanitizers had to fix some things --- Makefile | 5 ++++- TODO.md | 1 + include/strlist.h | 7 +++++++ src/executor.c | 6 +++++- src/notify.c | 15 ++++++++------- src/strlist.c | 36 ++++++++++++++++++++++++++++++++---- 6 files changed, 57 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 83f0962..741ec62 100644 --- a/Makefile +++ b/Makefile @@ -18,9 +18,12 @@ CFLAGS += -DSCI_NAME="\"$(NAME)\"" CFLAGS += -DSCI_DESCRIPTION="\"$(DESCRIPTION)\"" CFLAGS += -D_POSIX_C_SOURCE=2 CFLAGS += -D_GNU_SOURCE -CFLAGS += -Wall -Werror -std=c11 -g +CFLAGS += -Wall -Werror -std=c11 CFLAGS += -Iinclude CFLAGS += -lpthread -luuid +CFLAGS += -fsanitize=address +CFLAGS += -fsanitize=undefined +CFLAGS += -g .PHONY: all clean dist install diff --git a/TODO.md b/TODO.md index d4a5a66..2dbb1af 100644 --- a/TODO.md +++ b/TODO.md @@ -20,6 +20,7 @@ - [ ] Port this document to gitea issue tracking - [x] enable PATH-able programs and argv in the command section - [ ] custom environment variable passing. Something like `-e MY_TOKEN` ala docker-style + - [x] address sanitizers please. - [ ] Ninth things ninth, fix bugs, see below - [ ] Tenth things tenth, write manpages, choose license - [ ] Eleventh things Eleventh, polish diff --git a/include/strlist.h b/include/strlist.h index 2298ecd..95f29d9 100644 --- a/include/strlist.h +++ b/include/strlist.h @@ -48,4 +48,11 @@ void remove_strlist_node(strlist_node* node); // This function is not threadsafe. void clear_strlist(strlist_node* root); +// Convert a strlist to an array of strings. +// Note that this copies the strlist, you still have to free it. +// The array itself is NULL terminated, enabling you to iterate to the end. +// The array should be free'd, as well as each of the entries. +// Returns NULL if the provided strlist is empty. +char** strlist_to_array(strlist_node* root); + #endif diff --git a/src/executor.c b/src/executor.c index 08fe358..88324a8 100644 --- a/src/executor.c +++ b/src/executor.c @@ -24,7 +24,10 @@ void set_logdir(const char* logdir) { char* create_pipeline_id() { uuid_t uuid; uuid_generate(uuid); - char* pipeline_id = malloc(32); + // example uuid + // 662ddee9-ee7c-4d13-8999-a2604c6d12d6 + // it's 36 characters (+null) + char* pipeline_id = malloc(sizeof(char) * 37); uuid_unparse_lower(uuid, pipeline_id); return pipeline_id; } @@ -99,4 +102,5 @@ end: free(url); free(trigger); free(id); + free(data); } diff --git a/src/notify.c b/src/notify.c index 753ea7a..93291a3 100644 --- a/src/notify.c +++ b/src/notify.c @@ -18,6 +18,7 @@ #include "notify.h" #include "util.h" #include "log.h" +#include #define EV_SIZE sizeof(struct inotify_event) #define BUF_LEN EV_SIZE * 32 @@ -39,13 +40,13 @@ void listen_for_changes(const pipeline_conf* config, notify_callback callback) { assert(r != -1); for(int i = 0; i < r; ) { struct inotify_event* e = (struct inotify_event*)&buffer[i]; - pipeline_event ev; - ev.event = e; - ev.name = config->name; - ev.url = config->url; - ev.trigger = config->trigger; - ev.command = config->command; - callback(&ev); + pipeline_event* ev = malloc(sizeof(pipeline_event)); + ev->event = e; + ev->name = config->name; + ev->url = config->url; + ev->trigger = config->trigger; + ev->command = config->command; + callback(ev); i += EV_SIZE + e->len; } ASSERT_SYSCALL_SUCCESS(close(fd)); diff --git a/src/strlist.c b/src/strlist.c index 6fe70e3..be27000 100644 --- a/src/strlist.c +++ b/src/strlist.c @@ -53,10 +53,38 @@ void remove_strlist_node(strlist_node* node) { void clear_strlist(strlist_node* root) { strlist_node* cursor = root; while(cursor != NULL) { - cursor = cursor->next; - if(cursor->str) + if(cursor->str != NULL) free(cursor->str); - cursor->str = NULL; - free(cursor->previous); + strlist_node* prev = cursor; + cursor = cursor->next; + free(prev); } } + +size_t strlist_length(strlist_node* root) { + size_t result = 0; + strlist_node* cursor = root; + while(cursor != NULL) { + result++; + cursor = cursor->next; + } + return result; +} + +char** strlist_to_array(strlist_node* root) { + size_t len = strlist_length(root); + if(len <= 0) + return NULL; + char** result = malloc(sizeof(char*) * (len + 1)); + memset(result, len+1, (size_t)NULL); + strlist_node* cursor = root; + for(int i = 0; i < len; i++) { + if(cursor == NULL) + break; + if(cursor->str == NULL) + continue; + result[i] = strdup(cursor->str); + cursor = cursor->next; + } + return result; +} -- 2.52.0 From 484efc200a272c4ea8cc2fcce4f00565514fbaba Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Mon, 26 Aug 2024 19:51:12 +0200 Subject: [PATCH 06/18] feat: add environment variable passing --- TODO.md | 2 +- include/executor.h | 2 ++ include/util.h | 1 + src/executor.c | 55 +++++++++++++++++++++++++++++++++++++--------- src/main.c | 3 +++ src/strlist.c | 3 ++- src/util.c | 9 ++++++++ 7 files changed, 63 insertions(+), 12 deletions(-) diff --git a/TODO.md b/TODO.md index 2dbb1af..88d3dc6 100644 --- a/TODO.md +++ b/TODO.md @@ -19,7 +19,7 @@ - [ ] Eight things eight, try it out! - maybe even write the python webhook extension. - [ ] Port this document to gitea issue tracking - [x] enable PATH-able programs and argv in the command section - - [ ] custom environment variable passing. Something like `-e MY_TOKEN` ala docker-style + - [x] custom environment variable passing. Something like `-e MY_TOKEN` ala docker-style - [x] address sanitizers please. - [ ] Ninth things ninth, fix bugs, see below - [ ] Tenth things tenth, write manpages, choose license diff --git a/include/executor.h b/include/executor.h index 7c42747..c32b8b7 100644 --- a/include/executor.h +++ b/include/executor.h @@ -1,7 +1,9 @@ #ifndef SCI_EXECUTOR_H #define SCI_EXECUTOR_H +#include "strlist.h" void executor(void* pipeline_event); void set_logdir(const char* logdir); +void set_shared_environment(const strlist_node* root); #endif diff --git a/include/util.h b/include/util.h index d85d8e2..5589305 100644 --- a/include/util.h +++ b/include/util.h @@ -40,6 +40,7 @@ typedef void(*line_handler)(const char*); void per_line(const char* file, line_handler handler); char* join(const char* a, const char* b); +char* join3(const char* a, const char* b, const char* c); const char* skip_arg(const char* cp); char* skip_spaces(const char* str); diff --git a/src/executor.c b/src/executor.c index 88324a8..b822ed1 100644 --- a/src/executor.c +++ b/src/executor.c @@ -2,6 +2,7 @@ #include "log.h" #include "optional.h" #include "pipeline.h" +#include "strlist.h" #include "util.h" #include #include @@ -13,6 +14,11 @@ #include const char* log_dir = "./"; // NOTE: must end with a / +const strlist_node* shared_environment = NULL; + +void set_shared_environment(const strlist_node* root) { + shared_environment = root; +} void set_logdir(const char* logdir) { log_dir = logdir; @@ -49,6 +55,38 @@ optional_int open_logfile(const char* const pipeline_id) { return result; } +void add_joined_str(strlist_node* root, const char* a, const char* b) { + char* tmp = join(a, b); + add_str(tmp, root); + free(tmp); +} + +void add_env(strlist_node* root, const char* env) { + char* tmp = join3(env, "=", getenv(env)); + add_str(tmp, root); + free(tmp); +} + +char** create_environment(const pipeline_event* const e, const char* pipeline_id) { + char* tmp = join("PATH=", getenv("PATH")); // TODO: consider removing PATH default, since it can be done as -e PATH + strlist_node* env = create_strlist_node(tmp); + free(tmp); + add_joined_str(env, "SCI_PIPELINE_NAME=", e->name); + add_joined_str(env, "SCI_PIPELINE_URL=", e->url); + add_joined_str(env, "SCI_PIPELINE_TRIGGER=", e->trigger); + add_joined_str(env, "SCI_PIPELINE_ID=", pipeline_id); + if(shared_environment != NULL) { + const strlist_node* cursor = shared_environment; + while(cursor != NULL) { + add_env(env, cursor->str); + cursor = cursor->next; + } + } + char** envp = strlist_to_array(env); + clear_strlist(env); + return envp; +} + void executor(void* data) { // Create pipeline id char* pipeline_id = create_pipeline_id(); @@ -67,12 +105,7 @@ void executor(void* data) { posix_spawn_file_actions_adddup2(&actions, fd.value, STDOUT_FILENO); posix_spawn_file_actions_adddup2(&actions, fd.value, STDERR_FILENO); const pipeline_event* const e = data; - char* path = join("PATH=", getenv("PATH")); - char* name = join("SCI_PIPELINE_NAME=", e->name); - char* url = join("SCI_PIPELINE_URL=", e->url); - char* trigger = join("SCI_PIPELINE_TRIGGER=", e->trigger); - char* id = join("SCI_PIPELINE_ID=", pipeline_id); - char* envp[] = { path, name, url, trigger, id, NULL }; + char** envp = create_environment(e, pipeline_id); int argc; char** argv = argv_split(e->command, &argc); log_trace("executing pipeline %s with argv:", e->name); @@ -98,9 +131,11 @@ end: argv_free(argv); close(fd.value); free(pipeline_id); - free(name); - free(url); - free(trigger); - free(id); free(data); + char** cursor = envp; + while(*cursor != NULL) { + free(*cursor); + cursor++; + } + free(envp); } diff --git a/src/main.c b/src/main.c index b726a2f..e0b41a6 100644 --- a/src/main.c +++ b/src/main.c @@ -110,6 +110,9 @@ int main(int argc, char** argv) { fprintf(stderr, "no such file or directory %s\n", args.config_file.value); exit(EXIT_FAILURE); } + + if(args.environment_vars.has_value) + set_shared_environment(args.environment_vars.value); pool = threadpool_create(args.executors); per_line(args.config_file.value, &config_interpret_line); diff --git a/src/strlist.c b/src/strlist.c index be27000..3f26f57 100644 --- a/src/strlist.c +++ b/src/strlist.c @@ -76,7 +76,8 @@ char** strlist_to_array(strlist_node* root) { if(len <= 0) return NULL; char** result = malloc(sizeof(char*) * (len + 1)); - memset(result, len+1, (size_t)NULL); + for(int i = 0; i < len+1; i++) + result[i] = NULL; strlist_node* cursor = root; for(int i = 0; i < len; i++) { if(cursor == NULL) diff --git a/src/util.c b/src/util.c index 1f8973a..6bd6085 100644 --- a/src/util.c +++ b/src/util.c @@ -63,6 +63,15 @@ char* join(const char* a, const char* b) { return result; } +char* join3(const char* a, const char* b, const char* c) { + size_t alen = strlen(a); + size_t blen = strlen(b); + size_t clen = strlen(c); + char* result = malloc(alen + blen + clen + 1); + sprintf(result, "%s%s%s", a, b, c); + return result; +} + const char* skip_arg(const char* cp) { while(*cp && !isspace(*cp)) cp++; -- 2.52.0 From fd0eb59aae325cb04c47758a0bd36b08a0987abb Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Mon, 26 Aug 2024 20:53:28 +0200 Subject: [PATCH 07/18] fix: some cleanup --- TODO.md | 27 ++++----------------------- include/util.h | 1 + src/executor.c | 10 +++++----- src/sci.1 | 2 -- src/util.c | 10 ++++++++++ 5 files changed, 20 insertions(+), 30 deletions(-) diff --git a/TODO.md b/TODO.md index 88d3dc6..b576e75 100644 --- a/TODO.md +++ b/TODO.md @@ -8,7 +8,7 @@ - [x] Fourth things fourth, implement a prototype that reads a space-separated file and populates a struct. - [x] Fifth things fifth, implement a prototype that spawns a new thread that executes a shell command. - [x] Sixth things sixth, daemonize it! - - [ ] Seventh things seventh, package the sucker (arch, debian, alpine, docker) + - [x] Seventh things seventh, package the sucker (arch, debian, alpine, docker) - [x] archlinux - https://wiki.archlinux.org/title/Creating_packages - [x] debian @@ -16,12 +16,12 @@ - just use docker. - [-] ~~alpine~~ later. - [-] ~~docker~~ later. - - [ ] Eight things eight, try it out! - maybe even write the python webhook extension. - - [ ] Port this document to gitea issue tracking + - [x] Eight things eight, try it out! - maybe even write the python webhook extension. + - [x] Port this document to gitea issue tracking - [x] enable PATH-able programs and argv in the command section - [x] custom environment variable passing. Something like `-e MY_TOKEN` ala docker-style - [x] address sanitizers please. - - [ ] Ninth things ninth, fix bugs, see below + - [ ] Ninth things ninth, fix bugs, see https://git.gtz.dk/agj/sci/projects/1 - [ ] Tenth things tenth, write manpages, choose license - [ ] Eleventh things Eleventh, polish - [ ] Twelveth things last, release! @@ -71,25 +71,6 @@ alpine linux is using OpenRC (cool), which complicates things a little bit, but generally really well written. Otherwise, I am sure that both wiki.gentoo and wiki.archlinux have great pages too docker is super easy, just make a dockerfile - only concern is the trigger files. -#### Bugs / Missing Features - - [x] command output is being inherited. It should be piped into some random log-file - - [ ] pretty sure that `ctrl+c` / SIGINT is not graceful yet. - - [ ] missing license (heavily considering GPLv3) - - [ ] pipeline scripts should be executed in a unique `/tmp` dir - - [ ] Some way for third parties to see which pipelines are currently running and their status. - - Could be as simple as looking in the logs directory. - - How to mark a run as failed / success / warn? - - Third parties may need to extract artifacts. - or maybe the scripts themselves would upload artifacts? - - [ ] I am deliberately not using `Restart=on-failure` in the `scid.service` file because we are using `Type=exec` - and not `Type=notify` (yet) - which would require a `sd_notify` call of `READY=1` (see `man systemd.service`) - - [ ] Custom environment variables passed to the pipelines on invokation should be possible. - - [ ] Listener threads should be killed and restarted (worker pool should just chug along) when pipeline config file - has changed during runtime. Should be disableable with `--no-hot-reload-config` - i.e. on by default. - - [x] ~~`docker stop` is very slow. I am probably not handling signals properly yet.~~ native docker is abandonned - - [x] It seems that `-v 4` is segfaulting when running release builds, maybe the logger just cant find the source file? - Nope. I just wrote some bad code (inverted NULL check). - ### Note Regarding `inotify` usage From the manpage: ``` diff --git a/include/util.h b/include/util.h index 5589305..12ccf3b 100644 --- a/include/util.h +++ b/include/util.h @@ -41,6 +41,7 @@ void per_line(const char* file, line_handler handler); char* join(const char* a, const char* b); char* join3(const char* a, const char* b, const char* c); +char* join4(const char* a, const char* b, const char* c, const char* d); const char* skip_arg(const char* cp); char* skip_spaces(const char* str); diff --git a/src/executor.c b/src/executor.c index b822ed1..bee6bc3 100644 --- a/src/executor.c +++ b/src/executor.c @@ -8,12 +8,13 @@ #include #include #include +#include #include #include #include #include -const char* log_dir = "./"; // NOTE: must end with a / +const char* log_dir = "."; const strlist_node* shared_environment = NULL; void set_shared_environment(const strlist_node* root) { @@ -42,15 +43,13 @@ optional_int open_logfile(const char* const pipeline_id) { optional_int result; result.has_value = false; result.value = 0; - char* log_file = join(pipeline_id, ".log"); - char* log_filepath = join(log_dir, log_file); + char* log_filepath = join4(log_dir, "/", pipeline_id, ".log"); int fd = open(log_filepath, O_WRONLY | O_CREAT | O_TRUNC, 0644); if (fd != -1) { result.has_value = true; result.value = fd; } else perror("open"); - free(log_file); free(log_filepath); return result; } @@ -126,7 +125,8 @@ void executor(void* data) { log_info("{%s} (%s) exited with status %d", pipeline_id, e->name, status); char buf[32]; sprintf(buf, "exited with status %d", status); - write(fd.value, buf, strnlen(buf, 32)); + if(write(fd.value, buf, strnlen(buf, 32)) == -1) + perror("write"); end: argv_free(argv); close(fd.value); diff --git a/src/sci.1 b/src/sci.1 index c3af561..f7f3d2a 100644 --- a/src/sci.1 +++ b/src/sci.1 @@ -48,8 +48,6 @@ and each pipeline will have an associated pipeline trigger file that can be By default, pipeline triggers are placed in /tmp/sci but this can be overridden with the .OP -x. - - .SH EXAMPLES A simple example configuration file could look something like the following: diff --git a/src/util.c b/src/util.c index 6bd6085..f4bd13f 100644 --- a/src/util.c +++ b/src/util.c @@ -72,6 +72,16 @@ char* join3(const char* a, const char* b, const char* c) { return result; } +char* join4(const char* a, const char* b, const char* c, const char* d) { + size_t alen = strlen(a); + size_t blen = strlen(b); + size_t clen = strlen(c); + size_t dlen = strlen(d); + char* result = malloc(alen + blen + clen + dlen + 1); + sprintf(result, "%s%s%s%s", a, b, c, d); + return result; +} + const char* skip_arg(const char* cp) { while(*cp && !isspace(*cp)) cp++; -- 2.52.0 From 22075e17e17ff80b3f4a4b7e42c41bffdb314980 Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Thu, 29 Aug 2024 18:18:19 +0200 Subject: [PATCH 08/18] fix: remove PATH default env spawnp searches through PATH for you --- src/executor.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/executor.c b/src/executor.c index bee6bc3..dc53ca5 100644 --- a/src/executor.c +++ b/src/executor.c @@ -67,10 +67,9 @@ void add_env(strlist_node* root, const char* env) { } char** create_environment(const pipeline_event* const e, const char* pipeline_id) { - char* tmp = join("PATH=", getenv("PATH")); // TODO: consider removing PATH default, since it can be done as -e PATH + char* tmp = join("SCI_PIPELINE_NAME=", e->name); strlist_node* env = create_strlist_node(tmp); free(tmp); - add_joined_str(env, "SCI_PIPELINE_NAME=", e->name); add_joined_str(env, "SCI_PIPELINE_URL=", e->url); add_joined_str(env, "SCI_PIPELINE_TRIGGER=", e->trigger); add_joined_str(env, "SCI_PIPELINE_ID=", pipeline_id); @@ -113,7 +112,7 @@ void executor(void* data) { char arg0[PATH_MAX]; if(which(argv[0], arg0, PATH_MAX) == -1) goto end; - if(posix_spawn(&pid, arg0, &actions, NULL, argv, envp) != 0) { + if(posix_spawnp(&pid, arg0, &actions, NULL, argv, envp) != 0) { perror("posix_spawn"); goto end; // I know. The raptors have picked up the scent. I'll just have to mask it with more stinky code. } -- 2.52.0 From fa6ec315c83a93ffc7ea4bdf64e0d0527f81a145 Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Thu, 29 Aug 2024 20:51:11 +0200 Subject: [PATCH 09/18] feat: unique directory for pipelines Also, start using fork/chdir/exec idiom instead of posix_spawn, because as we all know: posix_spawn is stupid as a system call: https://lwn.net/Articles/360556/ --- include/cli.h | 1 + include/executor.h | 1 + src/cli.c | 15 ++++++++++++--- src/executor.c | 36 +++++++++++++++++++++++++++++------- src/main.c | 2 ++ 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/include/cli.h b/include/cli.h index f3f511c..348dda9 100644 --- a/include/cli.h +++ b/include/cli.h @@ -29,6 +29,7 @@ typedef struct { bool use_colors; optional_str log_file; optional_str pipeline_log_dir; + optional_str pipeline_cwd; optional_strlist environment_vars; } cli_options; diff --git a/include/executor.h b/include/executor.h index c32b8b7..9bc3887 100644 --- a/include/executor.h +++ b/include/executor.h @@ -4,6 +4,7 @@ void executor(void* pipeline_event); void set_logdir(const char* logdir); +void set_working_directory(const char* cwd); void set_shared_environment(const strlist_node* root); #endif diff --git a/src/cli.c b/src/cli.c index 317b936..fe4e010 100644 --- a/src/cli.c +++ b/src/cli.c @@ -51,6 +51,10 @@ cli_options new_options() { result.pipeline_log_dir.has_value = pipeline_log_dir != NULL; result.pipeline_log_dir.value = pipeline_log_dir; + char* pipeline_cwd = getenv("SCI_PIPELINE_CWD"); + result.pipeline_cwd.has_value = pipeline_cwd != NULL; + result.pipeline_cwd.value = pipeline_cwd; + char* environment_vars = getenv("SCI_PIPELINE_ENV_VARS"); if(environment_vars == NULL) { result.environment_vars.has_value = false; @@ -78,17 +82,18 @@ void destroy_options(cli_options v) { } // const char* log_dir = "."; +const char* cwd = "/tmp"; const strlist_node* shared_environment = NULL; void set_shared_environment(const strlist_node* root) { @@ -28,6 +29,13 @@ void set_logdir(const char* logdir) { mkdir(log_dir, 0700); } +void set_working_directory(const char* _cwd) { + cwd = _cwd; + struct stat st = {0}; + if(stat(cwd, &st) == -1) + mkdir(cwd, 0700); +} + char* create_pipeline_id() { uuid_t uuid; uuid_generate(uuid); @@ -98,10 +106,6 @@ void executor(void* data) { // spawn the process pid_t pid; - posix_spawn_file_actions_t actions; - posix_spawn_file_actions_init(&actions); - posix_spawn_file_actions_adddup2(&actions, fd.value, STDOUT_FILENO); - posix_spawn_file_actions_adddup2(&actions, fd.value, STDERR_FILENO); const pipeline_event* const e = data; char** envp = create_environment(e, pipeline_id); int argc; @@ -112,16 +116,34 @@ void executor(void* data) { char arg0[PATH_MAX]; if(which(argv[0], arg0, PATH_MAX) == -1) goto end; - if(posix_spawnp(&pid, arg0, &actions, NULL, argv, envp) != 0) { - perror("posix_spawn"); + + // fork / cwd / exec idiom + pid = fork(); + if(pid < 0) { + perror("fork"); goto end; // I know. The raptors have picked up the scent. I'll just have to mask it with more stinky code. } + + if(pid == 0) { + // child process + dup2(fd.value, STDOUT_FILENO); + dup2(fd.value, STDERR_FILENO); + char* pipeline_cwd = join3(cwd, "/", pipeline_id); + struct stat st = {0}; + if(stat(pipeline_cwd, &st) == -1) + mkdir(pipeline_cwd, 0700); + chdir(pipeline_cwd); + free(pipeline_cwd); + execvpe(arg0, argv, envp); + return; + } + log_info("{%s} (%s) spawned", pipeline_id, e->name); // Wait for process to complete int status; waitpid(pid, &status, 0); - log_info("{%s} (%s) exited with status %d", pipeline_id, e->name, status); + log_info("{%s} (%s) [pid=%d] exited with status %d", pipeline_id, e->name, pid, status); char buf[32]; sprintf(buf, "exited with status %d", status); if(write(fd.value, buf, strnlen(buf, 32)) == -1) diff --git a/src/main.c b/src/main.c index e0b41a6..d69f572 100644 --- a/src/main.c +++ b/src/main.c @@ -101,6 +101,8 @@ int main(int argc, char** argv) { if(args.pipeline_log_dir.has_value) set_logdir(args.pipeline_log_dir.value); + if(args.pipeline_cwd.has_value) + set_working_directory(args.pipeline_cwd.value); struct stat st = {0}; if(stat("/tmp/sci", &st) == -1) -- 2.52.0 From 4ba2362f5c3371f6b6d9af4b922b7cee8572f01b Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Sat, 31 Aug 2024 08:27:07 +0200 Subject: [PATCH 10/18] fix: slightly better trigger dir management --- include/pipeline.h | 1 + src/main.c | 38 +++++++++++++++----------------------- src/pipeline.c | 8 ++++++++ 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/include/pipeline.h b/include/pipeline.h index 0830077..19bf3a4 100644 --- a/include/pipeline.h +++ b/include/pipeline.h @@ -38,6 +38,7 @@ typedef struct { // create a new pipeline_conf struct instance based on a configuration line. optional_pipeline_conf pipeline_create(const char* config_line); +void pipeline_destroy(pipeline_conf* conf); void pipeline_register(pthread_t thread); void pipeline_loop(); diff --git a/src/main.c b/src/main.c index d69f572..3f71a01 100644 --- a/src/main.c +++ b/src/main.c @@ -27,6 +27,7 @@ #include threadpool* pool = NULL; +char* trigger_dir = "/tmp/sci"; void on_event(pipeline_event* const e) { if(!threadpool_add_work(pool, executor, (void*)e)) @@ -34,15 +35,11 @@ void on_event(pipeline_event* const e) { } void* listen_for_changes_thread(void* data) { - const pipeline_conf* conf = (const pipeline_conf*)data; + pipeline_conf* conf = (pipeline_conf*)data; while(1) // TODO: Should be while(sigint_has_not_been_caught) instead listen_for_changes(conf, &on_event); // We're now done with the config. - free(conf->name); - free(conf->url); - free(conf->trigger); - free(conf->command); - free(data); + pipeline_destroy(conf); return NULL; } @@ -64,14 +61,9 @@ void config_interpret_line(const char* line) { log_error("unable to register pipeline"); return; } - char* dest; - // NOTE: trigger names are allowed max 32 characters - dest = malloc(sizeof(*dest) * (9+33)); - dest[0] = '\0'; - strncat(dest, "/tmp/sci/", 10); - strncat(dest, conf.value->trigger, 33); + char* new_trigger_val = join3(trigger_dir, "/", conf.value->trigger); free(conf.value->trigger); - conf.value->trigger = dest; + conf.value->trigger = new_trigger_val; pthread_t t = spawn_listener(conf.value); pipeline_register(t); } @@ -98,21 +90,21 @@ int main(int argc, char** argv) { fprintf(stderr, "no pipeline config file provided see -h for usage\n"); exit(EXIT_FAILURE); } - - if(args.pipeline_log_dir.has_value) - set_logdir(args.pipeline_log_dir.value); - if(args.pipeline_cwd.has_value) - set_working_directory(args.pipeline_cwd.value); - - struct stat st = {0}; - if(stat("/tmp/sci", &st) == -1) - mkdir("/tmp/sci", 0700); - if(access(args.config_file.value, F_OK) != 0) { fprintf(stderr, "no such file or directory %s\n", args.config_file.value); exit(EXIT_FAILURE); } + if(args.pipeline_log_dir.has_value) + set_logdir(args.pipeline_log_dir.value); + + if(args.pipeline_cwd.has_value) + set_working_directory(args.pipeline_cwd.value); + + struct stat st = {0}; + if(stat(trigger_dir, &st) == -1) + mkdir(trigger_dir, 0700); + if(args.environment_vars.has_value) set_shared_environment(args.environment_vars.value); diff --git a/src/pipeline.c b/src/pipeline.c index 4e3f131..6cfda3b 100644 --- a/src/pipeline.c +++ b/src/pipeline.c @@ -68,6 +68,14 @@ optional_pipeline_conf pipeline_create(const char* config_line) { return result; } +void pipeline_destroy(pipeline_conf* conf) { + free(conf->name); + free(conf->url); + free(conf->trigger); + free(conf->command); + free(conf); +} + void pipeline_register(pthread_t thread) { if(root == NULL) { root = create_thread_node(thread); -- 2.52.0 From 7e80274784186c564e5b8f9a6ddd264b05f2d7d0 Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Mon, 2 Sep 2024 19:21:25 +0200 Subject: [PATCH 11/18] feat: add hot-reloading support Just launch and edit the pipeline.conf file. --- include/notify.h | 3 +++ src/main.c | 32 +++++++++++++++++++++++++++----- src/pipeline.c | 3 ++- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/include/notify.h b/include/notify.h index 205d6f3..2dc0fee 100644 --- a/include/notify.h +++ b/include/notify.h @@ -21,9 +21,12 @@ #include typedef void(*notify_callback)(pipeline_event* const); +typedef void(*config_change_callback)(); // Start listening for changes to the provided file. // Note that the `struct inotify_event*` provided is a managed pointer. void listen_for_changes(const pipeline_conf* config, notify_callback callback); +void listen_for_config_changes(const char* config_filepath, config_change_callback callback); + #endif diff --git a/src/main.c b/src/main.c index 3f71a01..51129dc 100644 --- a/src/main.c +++ b/src/main.c @@ -28,6 +28,7 @@ threadpool* pool = NULL; char* trigger_dir = "/tmp/sci"; +bool config_file_changed = false; void on_event(pipeline_event* const e) { if(!threadpool_add_work(pool, executor, (void*)e)) @@ -68,6 +69,18 @@ void config_interpret_line(const char* line) { pipeline_register(t); } +void on_config_file_changed() { + config_file_changed = true; + pipeline_cancel(); + log_info("config file changed, reloading..."); +} + +void* listen_for_config_changes_thread(void* data) { + while(1) + listen_for_config_changes((const char*)data, &on_config_file_changed); + return NULL; +} + int main(int argc, char** argv) { cli_options args = parse(argc, argv); log_settings settings; @@ -108,9 +121,18 @@ int main(int argc, char** argv) { if(args.environment_vars.has_value) set_shared_environment(args.environment_vars.value); - pool = threadpool_create(args.executors); - per_line(args.config_file.value, &config_interpret_line); - - pipeline_loop(); - threadpool_destroy(pool); + log_info("spawning trigger thread for config file"); + pthread_t conf_listener; + ASSERT_SYSCALL_SUCCESS(pthread_create(&conf_listener, NULL, &listen_for_config_changes_thread, (void*)args.config_file.value)); + pthread_setname_np(conf_listener, "sci-conf-listener"); + do { + config_file_changed = false; + worker_pool = threadpool_create(args.executors); + per_line(args.config_file.value, &config_interpret_line); + log_info("listening for pipeline invocations"); + pipeline_loop(); + } while(config_file_changed); + pthread_cancel(conf_listener); + threadpool_destroy(worker_pool); + destroy_options(args); } diff --git a/src/pipeline.c b/src/pipeline.c index 6cfda3b..6d5f923 100644 --- a/src/pipeline.c +++ b/src/pipeline.c @@ -48,7 +48,8 @@ optional_pipeline_conf pipeline_create(const char* config_line) { cursor += pmatch[0].rm_eo; } if(i != 4) { - log_error("invalid configuration!\nline is invalid: \"%s\""); + log_error("invalid configuration!"); + log_error("line is invalid: \"%s\"", config_line); for(int j = i-1; j >= 0; j--) free(opts[j]); return result; -- 2.52.0 From 5ba8a9f0636bf9b022e97ef549f5ef18f204ddcf Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Sat, 31 Aug 2024 09:09:01 +0200 Subject: [PATCH 12/18] fix: solve leak sanitizer on graceful shutdown I am testing by simply calling `pipeline_cancel` on the first pipeline execution. --- include/pipeline.h | 2 ++ src/executor.c | 2 +- src/main.c | 15 ++++++++++----- src/notify.c | 38 ++++++++++++++++++++++++++++++++------ src/pipeline.c | 18 ++++++++++++++++++ src/threadlist.c | 3 ++- 6 files changed, 65 insertions(+), 13 deletions(-) diff --git a/include/pipeline.h b/include/pipeline.h index 19bf3a4..b4d806a 100644 --- a/include/pipeline.h +++ b/include/pipeline.h @@ -38,8 +38,10 @@ typedef struct { // create a new pipeline_conf struct instance based on a configuration line. optional_pipeline_conf pipeline_create(const char* config_line); +void pipeline_event_destroy(pipeline_event* ev); void pipeline_destroy(pipeline_conf* conf); void pipeline_register(pthread_t thread); void pipeline_loop(); +void pipeline_cancel(); #endif diff --git a/src/executor.c b/src/executor.c index 719f3cf..90d7329 100644 --- a/src/executor.c +++ b/src/executor.c @@ -152,7 +152,7 @@ end: argv_free(argv); close(fd.value); free(pipeline_id); - free(data); + pipeline_event_destroy(data); char** cursor = envp; while(*cursor != NULL) { free(*cursor); diff --git a/src/main.c b/src/main.c index 51129dc..b714080 100644 --- a/src/main.c +++ b/src/main.c @@ -26,21 +26,26 @@ #include #include -threadpool* pool = NULL; +threadpool* worker_pool = NULL; char* trigger_dir = "/tmp/sci"; bool config_file_changed = false; void on_event(pipeline_event* const e) { - if(!threadpool_add_work(pool, executor, (void*)e)) + if(!threadpool_add_work(worker_pool, executor, (void*)e)) log_error("could not add work to the threadpool"); } +void listener_thread_cleanup(void* data) { + // We're now done with the config. + pipeline_destroy((pipeline_conf*)data); +} + void* listen_for_changes_thread(void* data) { pipeline_conf* conf = (pipeline_conf*)data; - while(1) // TODO: Should be while(sigint_has_not_been_caught) instead + pthread_cleanup_push(listener_thread_cleanup, conf); + while(1) listen_for_changes(conf, &on_event); - // We're now done with the config. - pipeline_destroy(conf); + pthread_cleanup_pop(1); return NULL; } diff --git a/src/notify.c b/src/notify.c index 93291a3..c10591f 100644 --- a/src/notify.c +++ b/src/notify.c @@ -24,7 +24,7 @@ #define BUF_LEN EV_SIZE * 32 void listen_for_changes(const pipeline_conf* config, notify_callback callback) { - // TODO: callback is a bit slow sometimes. We should also poll once after calling callback + // TODO: callback is potentially slow. We should also poll once after calling callback const char* filename = config->trigger; if(access(filename, F_OK) != 0) { log_trace("file does not exist yet, creating it."); @@ -37,17 +37,43 @@ void listen_for_changes(const pipeline_conf* config, notify_callback callback) { log_trace("listening for changes in file: %s", filename); char buffer[BUF_LEN]; int r = read(fd, buffer, BUF_LEN); - assert(r != -1); + if(r == -1) { + perror("read"); + return; + } for(int i = 0; i < r; ) { struct inotify_event* e = (struct inotify_event*)&buffer[i]; pipeline_event* ev = malloc(sizeof(pipeline_event)); ev->event = e; - ev->name = config->name; - ev->url = config->url; - ev->trigger = config->trigger; - ev->command = config->command; + ev->name = strdup(config->name); + ev->url = strdup(config->url); + ev->trigger = strdup(config->trigger); + ev->command = strdup(config->command); callback(ev); i += EV_SIZE + e->len; } ASSERT_SYSCALL_SUCCESS(close(fd)); } + +void listen_for_config_changes(const char* config_filepath, config_change_callback callback) { + if(access(config_filepath, F_OK) != 0) { + perror("access"); + return; + } + int fd = inotify_init(); + ASSERT_SYSCALL_SUCCESS(fd); + inotify_add_watch(fd, config_filepath, IN_ATTRIB); + log_trace("listening for changes in file: %s", config_filepath); + char buffer[BUF_LEN]; + int r = read(fd, buffer, BUF_LEN); + if(r == -1) { + perror("read"); + return; + } + assert(r != -1); + for(int i = 0; i < r; ) { + callback(); + i += EV_SIZE + ((struct inotify_event*)&buffer[i])->len;; + } + ASSERT_SYSCALL_SUCCESS(close(fd)); +} diff --git a/src/pipeline.c b/src/pipeline.c index 6d5f923..eb5284f 100644 --- a/src/pipeline.c +++ b/src/pipeline.c @@ -19,6 +19,7 @@ #include "pipeline.h" #include "threadlist.h" #include "util.h" +#include #include #include @@ -54,6 +55,7 @@ optional_pipeline_conf pipeline_create(const char* config_line) { free(opts[j]); return result; } + regfree(®); result.value = malloc(sizeof(pipeline_conf)); result.value->name = opts[0]; @@ -89,3 +91,19 @@ void pipeline_loop() { clear_thread_list(root); root = NULL; } + +void pipeline_cancel() { + pthread_list_node* cursor = root; + while(cursor != NULL) { + pthread_cancel(cursor->thread); + cursor = cursor->next; + } +} + +void pipeline_event_destroy(pipeline_event* ev) { + free(ev->name); + free(ev->trigger); + free(ev->url); + free(ev->command); + free(ev); +} diff --git a/src/threadlist.c b/src/threadlist.c index a3de7e7..9d81d2e 100644 --- a/src/threadlist.c +++ b/src/threadlist.c @@ -71,7 +71,8 @@ void clear_thread_list(pthread_list_node* root) { pthread_list_node* cursor = root; while(cursor != NULL) { pthread_join(cursor->thread, NULL); + pthread_list_node* prev = cursor; cursor = cursor->next; - free(cursor->previous); + free(prev); } } -- 2.52.0 From bdf3a54ba05874c519fe32b990c1e54b10565433 Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Sun, 8 Sep 2024 21:13:41 +0200 Subject: [PATCH 13/18] fix: add missing copyright notices --- include/executor.h | 17 +++++++++++++++++ src/executor.c | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/include/executor.h b/include/executor.h index 9bc3887..c7066e2 100644 --- a/include/executor.h +++ b/include/executor.h @@ -1,3 +1,20 @@ +/** + * sci - a simple ci system + Copyright (C) 2024 Asger Gitz-Johansen + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ #ifndef SCI_EXECUTOR_H #define SCI_EXECUTOR_H #include "strlist.h" diff --git a/src/executor.c b/src/executor.c index 90d7329..c5820e8 100644 --- a/src/executor.c +++ b/src/executor.c @@ -1,3 +1,20 @@ +/** + * sci - a simple ci system + Copyright (C) 2024 Asger Gitz-Johansen + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ #include "executor.h" #include "log.h" #include "optional.h" -- 2.52.0 From 99174939f5c0be8c924b59468bfbcefc62dd8385 Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Mon, 9 Sep 2024 07:21:09 +0200 Subject: [PATCH 14/18] feat: handle sigint properly --- src/main.c | 9 +++++++++ src/pipeline.c | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/src/main.c b/src/main.c index b714080..93c27c1 100644 --- a/src/main.c +++ b/src/main.c @@ -22,6 +22,7 @@ #include "pipeline.h" #include "threadpool.h" #include "util.h" +#include #include #include #include @@ -86,6 +87,12 @@ void* listen_for_config_changes_thread(void* data) { return NULL; } +void signal_handler(int signal) { + log_info("signal retrieved"); + if(signal == SIGINT) + pipeline_cancel(); +} + int main(int argc, char** argv) { cli_options args = parse(argc, argv); log_settings settings; @@ -94,6 +101,8 @@ int main(int argc, char** argv) { settings.out_file = args.log_file.has_value ? fopen(args.log_file.value, "w+") : stdout; log_init(settings); + signal(SIGINT, signal_handler); + if(args.help) { print_help(stdout, argv[0]); exit(EXIT_SUCCESS); diff --git a/src/pipeline.c b/src/pipeline.c index eb5284f..dbd41bb 100644 --- a/src/pipeline.c +++ b/src/pipeline.c @@ -26,6 +26,7 @@ pthread_list_node* root = NULL; optional_pipeline_conf pipeline_create(const char* config_line) { + log_trace("pipeline create"); optional_pipeline_conf result; result.has_value = false; const char* pattern = "[^[:blank:]]+|\"[^\"]*\""; @@ -72,6 +73,7 @@ optional_pipeline_conf pipeline_create(const char* config_line) { } void pipeline_destroy(pipeline_conf* conf) { + log_trace("pipeline destroy"); free(conf->name); free(conf->url); free(conf->trigger); @@ -80,6 +82,7 @@ void pipeline_destroy(pipeline_conf* conf) { } void pipeline_register(pthread_t thread) { + log_trace("pipeline register thread"); if(root == NULL) { root = create_thread_node(thread); return; @@ -88,11 +91,13 @@ void pipeline_register(pthread_t thread) { } void pipeline_loop() { + log_trace("pipeline loop"); clear_thread_list(root); root = NULL; } void pipeline_cancel() { + log_trace("cancelling pipeline"); pthread_list_node* cursor = root; while(cursor != NULL) { pthread_cancel(cursor->thread); @@ -101,6 +106,7 @@ void pipeline_cancel() { } void pipeline_event_destroy(pipeline_event* ev) { + log_trace("pipeline event destroy"); free(ev->name); free(ev->trigger); free(ev->url); -- 2.52.0 From aae388897e8c23e0c3da3d81d41df924ab9942e6 Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Wed, 11 Sep 2024 07:22:55 +0200 Subject: [PATCH 15/18] feat: add messagequeue api --- Makefile | 3 +- TODO.md | 1 + include/api.h | 27 ++++++++++++ include/pipeline.h | 3 +- include/util.h | 2 + src/api.c | 103 +++++++++++++++++++++++++++++++++++++++++++++ src/executor.c | 3 ++ src/main.c | 7 +++ src/pipeline.c | 10 +++++ src/sci.1 | 6 +-- src/util.c | 23 ++++++++++ 11 files changed, 183 insertions(+), 5 deletions(-) create mode 100644 include/api.h create mode 100644 src/api.c diff --git a/Makefile b/Makefile index 741ec62..bdc5072 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ CFLAGS += -D_POSIX_C_SOURCE=2 CFLAGS += -D_GNU_SOURCE CFLAGS += -Wall -Werror -std=c11 CFLAGS += -Iinclude -CFLAGS += -lpthread -luuid +CFLAGS += -lpthread -luuid -lrt CFLAGS += -fsanitize=address CFLAGS += -fsanitize=undefined CFLAGS += -g @@ -32,6 +32,7 @@ all: out/bin/sci out/obj/%.o: src/%.c | $(OBJDIR) $(CC) -c $? $(CFLAGS) -o $@ +OBJ += out/obj/api.o OBJ += out/obj/cli.o OBJ += out/obj/executor.o OBJ += out/obj/log.o diff --git a/TODO.md b/TODO.md index b576e75..dba99af 100644 --- a/TODO.md +++ b/TODO.md @@ -22,6 +22,7 @@ - [x] custom environment variable passing. Something like `-e MY_TOKEN` ala docker-style - [x] address sanitizers please. - [ ] Ninth things ninth, fix bugs, see https://git.gtz.dk/agj/sci/projects/1 + - [ ] docstring in all header files - [ ] Tenth things tenth, write manpages, choose license - [ ] Eleventh things Eleventh, polish - [ ] Twelveth things last, release! diff --git a/include/api.h b/include/api.h new file mode 100644 index 0000000..6b75679 --- /dev/null +++ b/include/api.h @@ -0,0 +1,27 @@ +#ifndef SCI_API_H +#define SCI_API_H + +// Start the api. This will also trigger an "sci started" event. +// Note that this is a blocking call. +void* api_start(void*); + +// Fork the process and have the child run the api. +void api_start_p(); + +// Destroy all listeners and release the message queue. +void api_destroy(); + +// Post a newline-separated string with pipeline_id entries +// of the currently running pipelines on the message queue. +void api_list_running_pipelines(); + +// Trigger a pipeline started event. +void api_pipeline_started(const char* pipeline_id, const char* name); + +// Trigger a pipeline ended event. +void api_pipeline_ended(const char* pipeline_id, const char* name, int exit_code); + +// Trigger an api started event. +void api_started(); + +#endif // !SCI_API_H diff --git a/include/pipeline.h b/include/pipeline.h index b4d806a..e0ebc9c 100644 --- a/include/pipeline.h +++ b/include/pipeline.h @@ -36,12 +36,13 @@ typedef struct { char* command; } pipeline_event; -// create a new pipeline_conf struct instance based on a configuration line. +// create a new `pipeline_conf` struct instance based on a configuration line. optional_pipeline_conf pipeline_create(const char* config_line); void pipeline_event_destroy(pipeline_event* ev); void pipeline_destroy(pipeline_conf* conf); void pipeline_register(pthread_t thread); void pipeline_loop(); void pipeline_cancel(); +int pipeline_count(); #endif diff --git a/include/util.h b/include/util.h index 12ccf3b..1f054f5 100644 --- a/include/util.h +++ b/include/util.h @@ -42,6 +42,8 @@ void per_line(const char* file, line_handler handler); char* join(const char* a, const char* b); char* join3(const char* a, const char* b, const char* c); char* join4(const char* a, const char* b, const char* c, const char* d); +char* join5(const char* a, const char* b, const char* c, const char* d, const char* e); +char* join6(const char* a, const char* b, const char* c, const char* d, const char* e, const char* f); const char* skip_arg(const char* cp); char* skip_spaces(const char* str); diff --git a/src/api.c b/src/api.c new file mode 100644 index 0000000..b849c1f --- /dev/null +++ b/src/api.c @@ -0,0 +1,103 @@ +#include "api.h" +#include "log.h" +#include "util.h" +#include +#include +#include +#include +#include +#include + +// TODO: Make sure to write a manpage for this api +#define LIST_REQ "list" +#define MQ_MAX_SIZE 8192 + +bool api_is_running = false; +mqd_t api_out; +mqd_t api_in; + +void api_handle_request(const char* request) { + log_trace("api request: '%s'", request); + + // list + if(strncmp(request, LIST_REQ, MQ_MAX_SIZE) == 0) { + api_list_running_pipelines(); + return; + } + + // else + log_error("unrecognized api request: '%s'", request); +} + +void* api_start(void* data) { + struct mq_attr attr; + attr.mq_flags = 0; + attr.mq_maxmsg = 10; + attr.mq_msgsize = 512; + attr.mq_curmsgs = 0; + + api_out = mq_open("/sci_tx", O_CREAT | O_WRONLY, 0666, &attr); + if(api_out == -1) { + perror("mq_open"); + return NULL; + } + + api_in = mq_open("/sci_rx", O_CREAT | O_RDONLY, 0666, &attr); // TODO: Consider some better mq names + if(api_in == -1) { + perror("mq_open"); + return NULL; + } + + api_started(); + + log_info("api listening for requests"); + char msg[MQ_MAX_SIZE]; + api_is_running = true; + while(api_is_running) { + memset(msg, '\0', MQ_MAX_SIZE); + if(mq_receive(api_in, msg, MQ_MAX_SIZE, NULL) == -1) { + perror("mq_receive"); + return NULL; + } + api_handle_request(msg); + } + return NULL; +} + +void api_start_p() { + pthread_t conf_listener; + ASSERT_SYSCALL_SUCCESS(pthread_create(&conf_listener, NULL, &api_start, (void*)NULL)); + pthread_setname_np(conf_listener, "sci-api"); +} + +void api_destroy() { + log_trace("closing api"); + api_is_running = false; + mq_unlink("/sci"); +} + +void api_list_running_pipelines() { + // TODO: you need a way of enumerating the pipeline ids before this can be implemented. + log_error("cannot list running pipelines yet, feature is work-in-progress."); +} + +void api_pipeline_started(const char* pipeline_id, const char* name) { + char* msg = join("pipeline_new ", pipeline_id); + if(mq_send(api_out, msg, strnlen(msg, 256), 1) == -1) + perror("mq_send"); + free(msg); +} + +void api_pipeline_ended(const char* pipeline_id, const char* name, int exit_code) { + char exit_code_str[64]; + sprintf(exit_code_str, " %d", exit_code); + char* msg = join6("pipeline_end ", pipeline_id, " ", name, " ", exit_code_str); + if(mq_send(api_out, msg, strnlen(msg, 256), 1) == -1) + perror("mq_send"); + free(msg); +} + +void api_started() { + if(mq_send(api_out, "sci_started", 12, 1) == -1) + perror("mq_send"); +} diff --git a/src/executor.c b/src/executor.c index c5820e8..5a5f0c6 100644 --- a/src/executor.c +++ b/src/executor.c @@ -16,6 +16,7 @@ along with this program. If not, see . */ #include "executor.h" +#include "api.h" #include "log.h" #include "optional.h" #include "pipeline.h" @@ -155,11 +156,13 @@ void executor(void* data) { return; } + api_pipeline_started(pipeline_id, e->name); log_info("{%s} (%s) spawned", pipeline_id, e->name); // Wait for process to complete int status; waitpid(pid, &status, 0); + api_pipeline_ended(pipeline_id, e->name, status); log_info("{%s} (%s) [pid=%d] exited with status %d", pipeline_id, e->name, pid, status); char buf[32]; sprintf(buf, "exited with status %d", status); diff --git a/src/main.c b/src/main.c index 93c27c1..6eabfb5 100644 --- a/src/main.c +++ b/src/main.c @@ -15,6 +15,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ +#include "api.h" #include "cli.h" #include "executor.h" #include "log.h" @@ -22,10 +23,12 @@ #include "pipeline.h" #include "threadpool.h" #include "util.h" +#include #include #include #include #include +#include threadpool* worker_pool = NULL; char* trigger_dir = "/tmp/sci"; @@ -117,6 +120,7 @@ int main(int argc, char** argv) { fprintf(stderr, "no pipeline config file provided see -h for usage\n"); exit(EXIT_FAILURE); } + if(access(args.config_file.value, F_OK) != 0) { fprintf(stderr, "no such file or directory %s\n", args.config_file.value); exit(EXIT_FAILURE); @@ -134,6 +138,8 @@ int main(int argc, char** argv) { if(args.environment_vars.has_value) set_shared_environment(args.environment_vars.value); + + api_start_p(); log_info("spawning trigger thread for config file"); pthread_t conf_listener; @@ -149,4 +155,5 @@ int main(int argc, char** argv) { pthread_cancel(conf_listener); threadpool_destroy(worker_pool); destroy_options(args); + api_destroy(); } diff --git a/src/pipeline.c b/src/pipeline.c index dbd41bb..824127d 100644 --- a/src/pipeline.c +++ b/src/pipeline.c @@ -113,3 +113,13 @@ void pipeline_event_destroy(pipeline_event* ev) { free(ev->command); free(ev); } + +int pipeline_count() { + int result = 0; + pthread_list_node* cursor = root; + while(cursor != NULL) { + cursor = cursor->next; + result++; + } + return result; +} diff --git a/src/sci.1 b/src/sci.1 index f7f3d2a..443b147 100644 --- a/src/sci.1 +++ b/src/sci.1 @@ -42,7 +42,7 @@ The operation of is configured through a .I pipelines.conf configuration file (see -.I sci(7) +.BR sci (5) for configuration language details) and each pipeline will have an associated pipeline trigger file that can be By default, pipeline triggers are placed in /tmp/sci but this can be overridden with the @@ -85,5 +85,5 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . -.SH SEE ALSO -\" TODO: write sci(7) +.SH "SEE ALSO" +.BR sci (5), diff --git a/src/util.c b/src/util.c index f4bd13f..9c8e147 100644 --- a/src/util.c +++ b/src/util.c @@ -82,6 +82,29 @@ char* join4(const char* a, const char* b, const char* c, const char* d) { return result; } +char* join5(const char* a, const char* b, const char* c, const char* d, const char* e) { + size_t alen = strlen(a); + size_t blen = strlen(b); + size_t clen = strlen(c); + size_t dlen = strlen(d); + size_t elen = strlen(e); + char* result = malloc(alen + blen + clen + dlen + elen + 1); + sprintf(result, "%s%s%s%s%s", a, b, c, d, e); + return result; +} + +char* join6(const char* a, const char* b, const char* c, const char* d, const char* e, const char* f) { + size_t alen = strlen(a); + size_t blen = strlen(b); + size_t clen = strlen(c); + size_t dlen = strlen(d); + size_t elen = strlen(e); + size_t flen = strlen(f); + char* result = malloc(alen + blen + clen + dlen + elen + flen + 1); + sprintf(result, "%s%s%s%s%s%s", a, b, c, d, e, f); + return result; +} + const char* skip_arg(const char* cp) { while(*cp && !isspace(*cp)) cp++; -- 2.52.0 From e1ab3f28d6fdd5118049d7a4ebc8f576d63b15a2 Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Wed, 18 Sep 2024 19:46:27 +0200 Subject: [PATCH 16/18] fix: log timestamps are now updated --- src/log.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/log.c b/src/log.c index e48c3a9..0a8145a 100644 --- a/src/log.c +++ b/src/log.c @@ -78,6 +78,7 @@ void log_log(const char* file, int line, int level, const char* fmt, ...) { g_log_tm = localtime(&t); } strftime(timestamp, sizeof(timestamp), "%H:%M:%S", g_log_tm); + g_log_tm = NULL; const char* level_color = get_level_color(level); const char* level_name = get_level_name(level); -- 2.52.0 From a5618d3c5fa5d49637b0270ea1dbe5ebad4c96cb Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Tue, 16 Jun 2026 22:44:25 +0200 Subject: [PATCH 17/18] fix: validate input in wget-and-sci.sh --- scripts/wget-and-sci.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/wget-and-sci.sh b/scripts/wget-and-sci.sh index 1b79fc3..5e7585e 100755 --- a/scripts/wget-and-sci.sh +++ b/scripts/wget-and-sci.sh @@ -1,13 +1,18 @@ #!/bin/sh # NOTE: This script assumes that the url is a .tar.gz file. -# TODO: check if $# is >= 1 and give a warning that the extract dir should be provided. +if [ "$#" -lt 1 ]; then + printf '%s\n' "usage: $0 EXTRACT_DIR" >&2 + printf '%s\n' "EXTRACT_DIR is the directory to enter after extracting the archive." >&2 + exit 2 +fi + set -ex # print all that we're doing (no need for echo's) env tmpdir=$(mktemp -d) wget "$SCI_PIPELINE_URL" -P "$tmpdir" cd "$tmpdir" tar xf *.tar.gz -cd $1 +cd "$1" sh .sci.sh cd - rm -rf "$tmpdir" -- 2.52.0 From 80857e0f0aa1d0f8ff50bb7219f1924ee24c14c9 Mon Sep 17 00:00:00 2001 From: Asger Gitz-Johansen Date: Tue, 16 Jun 2026 22:48:04 +0200 Subject: [PATCH 18/18] wip: fix existing todos Co-authored-by: Github CoPilot --- .sci.sh | 7 +-- Makefile | 4 ++ README.md | 6 ++- include/threadlist.h | 1 + src/api.c | 102 ++++++++++++++++++++++++++++++++++++------- src/notify.c | 56 +++++++++++++++--------- src/sci-api.7 | 35 +++++++++++++++ src/sci.1 | 1 + src/threadlist.c | 30 +++++++++---- src/threadpool.c | 35 +++++++++------ 10 files changed, 212 insertions(+), 65 deletions(-) create mode 100644 src/sci-api.7 diff --git a/.sci.sh b/.sci.sh index f9a8444..850a563 100755 --- a/.sci.sh +++ b/.sci.sh @@ -2,6 +2,8 @@ set -e echo ">>> checking if required environment is set..." test -n "$DOCKER_TOKEN" +: "${PACKAGE_REGISTRY_USER:=agj}" +export PACKAGE_REGISTRY_USER which make echo ">>> compiling..." @@ -30,7 +32,7 @@ echo ">>> building debbuilder image..." docker build -t debbuilder -f deb-builder.dockerfile . echo ">>> building .deb in debbuilder docker image..." -docker run --rm -it -v .:/src -e VERSION -e DOCKER_TOKEN debbuilder sh -c '\ +docker run --rm -it -v .:/src -e VERSION -e DOCKER_TOKEN -e PACKAGE_REGISTRY_USER debbuilder sh -c '\ cd && \ mkdir -p artifacts && \ cp /src/sci-$VERSION.tar.gz . && \ @@ -47,8 +49,7 @@ docker run --rm -it -v .:/src -e VERSION -e DOCKER_TOKEN debbuilder sh -c '\ cp ../*.tar.xz ~/artifacts && \ cp ../*.tar.gz ~/artifacts && \ cd && \ - curl --user agj:$DOCKER_TOKEN \ + curl --user "$PACKAGE_REGISTRY_USER:$DOCKER_TOKEN" \ --upload-file sci_$VERSION-1_amd64.deb \ "https://git.gtz.dk/api/packages/agj/debian/pool/bionic/main/upload" ' -# TODO: push-user should be some sci-bot or something, not your account. This will do for now though diff --git a/Makefile b/Makefile index bdc5072..a3b2814 100644 --- a/Makefile +++ b/Makefile @@ -85,6 +85,9 @@ install: out/bin/sci mkdir -p $(DESTDIR)$(MANPREFIX)/man1 sed "s/VERSION/$(VERSION)/g" < src/sci.1 > $(DESTDIR)$(MANPREFIX)/man1/sci.1 chmod 644 $(DESTDIR)$(MANPREFIX)/man1/sci.1 + mkdir -p $(DESTDIR)$(MANPREFIX)/man7 + sed "s/VERSION/$(VERSION)/g" < src/sci-api.7 > $(DESTDIR)$(MANPREFIX)/man7/sci-api.7 + chmod 644 $(DESTDIR)$(MANPREFIX)/man7/sci-api.7 uninstall: # uninstall binaries @@ -93,3 +96,4 @@ uninstall: # uninstall services (only if system is using systemd though) # uninstall manpages rm -f $(DESTDIR)$(MANPREFIX)/man1/sci.1 + rm -f $(DESTDIR)$(MANPREFIX)/man7/sci-api.7 diff --git a/README.md b/README.md index 72fa79f..79f9065 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,9 @@ sudo make install PREFIX=/some/path ### Arch Linux It is recommended that you use an arch linux distribution when building. - +The `arch-builder.dockerfile` image provides a repeatable build environment. ```sh +docker build -t archbuilder -f arch-builder.dockerfile . ``` ### Debian @@ -65,7 +66,8 @@ docker build -t debbuilder deb-builder.dockerfile . ``` ## Brainstorm -If you dont want to congest your CI server. Too bad. Write faster ci suites. (TODO: implement runners) +If you dont want to congest your CI server, keep pipeline scripts fast or run them on separate machines. +Runner orchestration is intentionally outside the current core daemon. I would like to try to avoid writing a million REST APIs, as that just results in bloat usually. diff --git a/include/threadlist.h b/include/threadlist.h index ce62459..0590cfd 100644 --- a/include/threadlist.h +++ b/include/threadlist.h @@ -22,6 +22,7 @@ // doubly linked list implementation for managing threads typedef struct pthread_list_node { pthread_t thread; + pthread_mutex_t mutex; struct pthread_list_node* previous; struct pthread_list_node* next; } pthread_list_node; diff --git a/src/api.c b/src/api.c index b849c1f..912c275 100644 --- a/src/api.c +++ b/src/api.c @@ -8,13 +8,61 @@ #include #include -// TODO: Make sure to write a manpage for this api #define LIST_REQ "list" #define MQ_MAX_SIZE 8192 +#define SCI_API_EVENTS_QUEUE "/sci_events" +#define SCI_API_REQUESTS_QUEUE "/sci_requests" bool api_is_running = false; -mqd_t api_out; -mqd_t api_in; +mqd_t api_out = (mqd_t)-1; +mqd_t api_in = (mqd_t)-1; + +typedef struct api_pipeline_node { + char* pipeline_id; + char* name; + struct api_pipeline_node* next; +} api_pipeline_node; + +pthread_mutex_t api_pipeline_mutex = PTHREAD_MUTEX_INITIALIZER; +api_pipeline_node* api_running_pipelines = NULL; + +static void api_send(const char* msg) { + if(mq_send(api_out, msg, strnlen(msg, MQ_MAX_SIZE), 1) == -1) + perror("mq_send"); +} + +static void api_register_pipeline(const char* pipeline_id, const char* name) { + api_pipeline_node* node = malloc(sizeof(api_pipeline_node)); + node->pipeline_id = strdup(pipeline_id); + node->name = strdup(name); + + pthread_mutex_lock(&api_pipeline_mutex); + node->next = api_running_pipelines; + api_running_pipelines = node; + pthread_mutex_unlock(&api_pipeline_mutex); +} + +static void api_unregister_pipeline(const char* pipeline_id) { + pthread_mutex_lock(&api_pipeline_mutex); + api_pipeline_node* previous = NULL; + api_pipeline_node* cursor = api_running_pipelines; + while(cursor != NULL) { + if(strncmp(cursor->pipeline_id, pipeline_id, MQ_MAX_SIZE) == 0) { + if(previous == NULL) + api_running_pipelines = cursor->next; + else + previous->next = cursor->next; + free(cursor->pipeline_id); + free(cursor->name); + free(cursor); + pthread_mutex_unlock(&api_pipeline_mutex); + return; + } + previous = cursor; + cursor = cursor->next; + } + pthread_mutex_unlock(&api_pipeline_mutex); +} void api_handle_request(const char* request) { log_trace("api request: '%s'", request); @@ -33,16 +81,16 @@ void* api_start(void* data) { struct mq_attr attr; attr.mq_flags = 0; attr.mq_maxmsg = 10; - attr.mq_msgsize = 512; + attr.mq_msgsize = MQ_MAX_SIZE; attr.mq_curmsgs = 0; - api_out = mq_open("/sci_tx", O_CREAT | O_WRONLY, 0666, &attr); + api_out = mq_open(SCI_API_EVENTS_QUEUE, O_CREAT | O_WRONLY, 0666, &attr); if(api_out == -1) { perror("mq_open"); return NULL; } - api_in = mq_open("/sci_rx", O_CREAT | O_RDONLY, 0666, &attr); // TODO: Consider some better mq names + api_in = mq_open(SCI_API_REQUESTS_QUEUE, O_CREAT | O_RDONLY, 0666, &attr); if(api_in == -1) { perror("mq_open"); return NULL; @@ -73,31 +121,53 @@ void api_start_p() { void api_destroy() { log_trace("closing api"); api_is_running = false; - mq_unlink("/sci"); + if(api_in != (mqd_t)-1) + mq_close(api_in); + if(api_out != (mqd_t)-1) + mq_close(api_out); + mq_unlink(SCI_API_REQUESTS_QUEUE); + mq_unlink(SCI_API_EVENTS_QUEUE); } void api_list_running_pipelines() { - // TODO: you need a way of enumerating the pipeline ids before this can be implemented. - log_error("cannot list running pipelines yet, feature is work-in-progress."); + char msg[MQ_MAX_SIZE]; + msg[0] = '\0'; + size_t used = 0; + + pthread_mutex_lock(&api_pipeline_mutex); + api_pipeline_node* cursor = api_running_pipelines; + while(cursor != NULL) { + int written = snprintf(msg + used, MQ_MAX_SIZE - used, "%s%s", used == 0 ? "" : "\n", cursor->pipeline_id); + if(written < 0) + break; + if((size_t)written >= MQ_MAX_SIZE - used) { + log_error("running pipeline list exceeds api message size"); + msg[MQ_MAX_SIZE - 1] = '\0'; + break; + } + used += written; + cursor = cursor->next; + } + pthread_mutex_unlock(&api_pipeline_mutex); + api_send(msg); } void api_pipeline_started(const char* pipeline_id, const char* name) { + api_register_pipeline(pipeline_id, name); char* msg = join("pipeline_new ", pipeline_id); - if(mq_send(api_out, msg, strnlen(msg, 256), 1) == -1) - perror("mq_send"); + api_send(msg); free(msg); } void api_pipeline_ended(const char* pipeline_id, const char* name, int exit_code) { + api_unregister_pipeline(pipeline_id); char exit_code_str[64]; - sprintf(exit_code_str, " %d", exit_code); + snprintf(exit_code_str, sizeof(exit_code_str), "%d", exit_code); char* msg = join6("pipeline_end ", pipeline_id, " ", name, " ", exit_code_str); - if(mq_send(api_out, msg, strnlen(msg, 256), 1) == -1) - perror("mq_send"); + api_send(msg); free(msg); } void api_started() { - if(mq_send(api_out, "sci_started", 12, 1) == -1) - perror("mq_send"); + api_send("sci_started"); } diff --git a/src/notify.c b/src/notify.c index c10591f..80d53a8 100644 --- a/src/notify.c +++ b/src/notify.c @@ -18,30 +18,15 @@ #include "notify.h" #include "util.h" #include "log.h" +#include +#include #include #define EV_SIZE sizeof(struct inotify_event) #define BUF_LEN EV_SIZE * 32 -void listen_for_changes(const pipeline_conf* config, notify_callback callback) { - // TODO: callback is potentially slow. We should also poll once after calling callback - const char* filename = config->trigger; - if(access(filename, F_OK) != 0) { - log_trace("file does not exist yet, creating it."); - FILE* f = fopen(filename, "w+"); - fclose(f); - } - int fd = inotify_init(); - ASSERT_SYSCALL_SUCCESS(fd); - inotify_add_watch(fd, filename, IN_ATTRIB); - log_trace("listening for changes in file: %s", filename); - char buffer[BUF_LEN]; - int r = read(fd, buffer, BUF_LEN); - if(r == -1) { - perror("read"); - return; - } - for(int i = 0; i < r; ) { +static void dispatch_pipeline_events(const pipeline_conf* config, notify_callback callback, char* buffer, ssize_t len) { + for(int i = 0; i < len; ) { struct inotify_event* e = (struct inotify_event*)&buffer[i]; pipeline_event* ev = malloc(sizeof(pipeline_event)); ev->event = e; @@ -52,6 +37,35 @@ void listen_for_changes(const pipeline_conf* config, notify_callback callback) { callback(ev); i += EV_SIZE + e->len; } +} + +void listen_for_changes(const pipeline_conf* config, notify_callback callback) { + const char* filename = config->trigger; + if(access(filename, F_OK) != 0) { + log_trace("file does not exist yet, creating it."); + FILE* f = fopen(filename, "w+"); + fclose(f); + } + int fd = inotify_init(); + ASSERT_SYSCALL_SUCCESS(fd); + ASSERT_SYSCALL_SUCCESS(inotify_add_watch(fd, filename, IN_ATTRIB)); + log_trace("listening for changes in file: %s", filename); + char buffer[BUF_LEN]; + ssize_t r = read(fd, buffer, BUF_LEN); + if(r == -1) { + perror("read"); + return; + } + dispatch_pipeline_events(config, callback, buffer, r); + + int flags = fcntl(fd, F_GETFL, 0); + if(flags != -1 && fcntl(fd, F_SETFL, flags | O_NONBLOCK) != -1) { + while((r = read(fd, buffer, BUF_LEN)) > 0) + dispatch_pipeline_events(config, callback, buffer, r); + if(r == -1 && errno != EAGAIN && errno != EWOULDBLOCK) + perror("read"); + } + ASSERT_SYSCALL_SUCCESS(close(fd)); } @@ -62,10 +76,10 @@ void listen_for_config_changes(const char* config_filepath, config_change_callba } int fd = inotify_init(); ASSERT_SYSCALL_SUCCESS(fd); - inotify_add_watch(fd, config_filepath, IN_ATTRIB); + ASSERT_SYSCALL_SUCCESS(inotify_add_watch(fd, config_filepath, IN_ATTRIB)); log_trace("listening for changes in file: %s", config_filepath); char buffer[BUF_LEN]; - int r = read(fd, buffer, BUF_LEN); + ssize_t r = read(fd, buffer, BUF_LEN); if(r == -1) { perror("read"); return; diff --git a/src/sci-api.7 b/src/sci-api.7 new file mode 100644 index 0000000..3b78f71 --- /dev/null +++ b/src/sci-api.7 @@ -0,0 +1,35 @@ +.TH sci-api 7 2024-08-17 "VERSION" "Simple CI manual" + +.SH NAME +sci-api - POSIX message queue interface for sci + +.SH DESCRIPTION +.B sci +exposes a small local API through POSIX message queues. +Clients send requests to +.I /sci_requests +and receive events and responses from +.I /sci_events. + +.SH REQUESTS +.TP +.B list +Return the currently running pipeline IDs as a newline-separated message. +An empty message means no pipelines are running. + +.SH EVENTS +.TP +.B sci_started +The daemon API thread has started. +.TP +.BI pipeline_new " pipeline_id" +A pipeline process has started. +.TP +.BI pipeline_end " pipeline_id name exit_status" +A pipeline process has exited. + +.SH LIMITS +Messages are limited to 8192 bytes. + +.SH SEE ALSO +.BR sci (1) diff --git a/src/sci.1 b/src/sci.1 index 443b147..83094f6 100644 --- a/src/sci.1 +++ b/src/sci.1 @@ -87,3 +87,4 @@ along with this program. If not, see . .SH "SEE ALSO" .BR sci (5), +.BR sci-api (7), diff --git a/src/threadlist.c b/src/threadlist.c index 9d81d2e..f76970e 100644 --- a/src/threadlist.c +++ b/src/threadlist.c @@ -18,19 +18,23 @@ #include "threadlist.h" #include -// TODO: mutex should really be per-list. -pthread_mutex_t threadlist_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_list_node* threadlist_root(pthread_list_node* node) { + while(node != NULL && node->previous != NULL) + node = node->previous; + return node; +} pthread_list_node* create_thread_node(pthread_t thread) { pthread_list_node* new = malloc(sizeof(pthread_list_node)); new->previous = NULL; new->next = NULL; new->thread = thread; + pthread_mutex_init(&new->mutex, NULL); return new; } pthread_list_node* add_thread(pthread_t thread, pthread_list_node* root) { - pthread_mutex_lock(&threadlist_mutex); + pthread_mutex_lock(&root->mutex); pthread_list_node* cursor = root; while(cursor->next != NULL) cursor = cursor->next; @@ -38,41 +42,49 @@ pthread_list_node* add_thread(pthread_t thread, pthread_list_node* root) { new->previous = cursor; new->next = NULL; new->thread = thread; + pthread_mutex_init(&new->mutex, NULL); cursor->next = new; - pthread_mutex_unlock(&threadlist_mutex); + pthread_mutex_unlock(&root->mutex); return new; } pthread_list_node* add_thread_node(pthread_list_node* root, pthread_list_node* node) { - pthread_mutex_lock(&threadlist_mutex); + pthread_mutex_lock(&root->mutex); pthread_list_node* cursor = root; while(cursor->next != NULL) cursor = cursor->next; node->previous = cursor; node->next = NULL; cursor->next = node; - pthread_mutex_unlock(&threadlist_mutex); + pthread_mutex_unlock(&root->mutex); return node; } void remove_thread_node(pthread_list_node* node) { - pthread_mutex_lock(&threadlist_mutex); + if(node == NULL) + return; + pthread_list_node* root = threadlist_root(node); + pthread_mutex_lock(&root->mutex); pthread_list_node* prev = node->previous; pthread_list_node* next = node->next; - free(node); if(prev != NULL) prev->next = next; if(next != NULL) next->previous = prev; - pthread_mutex_unlock(&threadlist_mutex); + pthread_mutex_unlock(&root->mutex); + pthread_mutex_destroy(&node->mutex); + free(node); } void clear_thread_list(pthread_list_node* root) { + if(root == NULL) + return; pthread_list_node* cursor = root; while(cursor != NULL) { pthread_join(cursor->thread, NULL); pthread_list_node* prev = cursor; cursor = cursor->next; + pthread_mutex_destroy(&prev->mutex); free(prev); } } diff --git a/src/threadpool.c b/src/threadpool.c index 0cf9aad..8f19191 100644 --- a/src/threadpool.c +++ b/src/threadpool.c @@ -41,28 +41,33 @@ static void threadpool_work_destroy(threadpool_work* work) { free(work); } -static threadpool_work* threadpool_work_pop(threadpool* pool) { - threadpool_work* work; +static optional_threadpool_work threadpool_work_pop(threadpool* pool) { + optional_threadpool_work result; + result.value = NULL; + result.has_value = false; if (pool == NULL) - return NULL; + return result; - work = pool->work_first; + threadpool_work* work = pool->work_first; if (work == NULL) - return NULL; // TODO: This should propbably be using optionals + return result; if (work->next == NULL) { pool->work_first = NULL; pool->work_last = NULL; - return work; + result.value = work; + result.has_value = true; + return result; } pool->work_first = work->next; - return work; + result.value = work; + result.has_value = true; + return result; } static void* threadpool_worker(void* arg) { log_trace("threadpool worker spawned"); threadpool* pool = arg; - threadpool_work* work; while(true) { // Wait for work pthread_mutex_lock(&(pool->work_mutex)); @@ -70,18 +75,20 @@ static void* threadpool_worker(void* arg) { pthread_cond_wait(&(pool->work_cond), &(pool->work_mutex)); if (pool->stop) break; - work = threadpool_work_pop(pool); - pool->working_count++; + optional_threadpool_work work = threadpool_work_pop(pool); + if(work.has_value) + pool->working_count++; pthread_mutex_unlock(&(pool->work_mutex)); // Do the work - if (work != NULL) { - work->func(work->arg); - threadpool_work_destroy(work); + if (work.has_value) { + work.value->func(work.value->arg); + threadpool_work_destroy(work.value); } pthread_mutex_lock(&(pool->work_mutex)); - pool->working_count--; + if(work.has_value) + pool->working_count--; if (!pool->stop && pool->working_count == 0 && pool->work_first == NULL) pthread_cond_signal(&(pool->working_cond)); pthread_mutex_unlock(&(pool->work_mutex)); -- 2.52.0