diff --git a/.dockerfile b/.dockerfile deleted file mode 100644 index b3174e9..0000000 --- a/.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -# TODO: use alpine when available -FROM debian:12-slim -ADD artifacts.tar.gz /install -RUN dpkg -i /install/artifacts/sci_*-1_amd64.deb -RUN rm -rf /install -ENTRYPOINT ["sci"] diff --git a/.sci.sh b/.sci.sh index a358d9a..850a563 100755 --- a/.sci.sh +++ b/.sci.sh @@ -2,6 +2,9 @@ set -e echo ">>> checking if required environment is set..." test -n "$DOCKER_TOKEN" +: "${PACKAGE_REGISTRY_USER:=agj}" +export PACKAGE_REGISTRY_USER +which make echo ">>> compiling..." make @@ -12,7 +15,7 @@ make dist SRC_SHA256=$(sha256sum "sci-${VERSION}.tar.gz" | awk '{ print $1 }') sed "s/SRC_SHA256/${SRC_SHA256}/g" < PKGBUILD.in > PKGBUILD -# arch +# # arch echo ">>> building archbuilder image..." docker build -t archbuilder -f arch-builder.dockerfile . @@ -29,7 +32,7 @@ echo ">>> building debbuilder image..." docker build -t debbuilder -f deb-builder.dockerfile . echo ">>> building .deb in debbuilder docker image..." -docker run --rm -it -v .:/src -e VERSION debbuilder sh -c '\ +docker run --rm -it -v .:/src -e VERSION -e DOCKER_TOKEN -e PACKAGE_REGISTRY_USER debbuilder sh -c '\ cd && \ mkdir -p artifacts && \ cp /src/sci-$VERSION.tar.gz . && \ @@ -46,14 +49,7 @@ docker run --rm -it -v .:/src -e VERSION debbuilder sh -c '\ cp ../*.tar.xz ~/artifacts && \ cp ../*.tar.gz ~/artifacts && \ cd && \ - tar czf /src/artifacts.tar.gz artifacts + curl --user "$PACKAGE_REGISTRY_USER:$DOCKER_TOKEN" \ + --upload-file sci_$VERSION-1_amd64.deb \ + "https://git.gtz.dk/api/packages/agj/debian/pool/bionic/main/upload" ' - -echo ">>> building sci docker image..." -export OWNER="git.gtz.dk/agj" -docker build -t ${OWNER}/sci:${VERSION} -t ${OWNER}/sci:latest -f .dockerfile . - -echo ">>> pushing latest docker image..." -# TODO: user should be some sci-bot or something, not your account. This will do for now though -docker login git.gtz.dk -u agj -p "$DOCKER_TOKEN" -docker push ${OWNER}/sci:latest diff --git a/Makefile b/Makefile index 4bf0d8d..a3b2814 100644 --- a/Makefile +++ b/Makefile @@ -18,9 +18,12 @@ CFLAGS += -DSCI_NAME="\"$(NAME)\"" CFLAGS += -DSCI_DESCRIPTION="\"$(DESCRIPTION)\"" CFLAGS += -D_POSIX_C_SOURCE=2 CFLAGS += -D_GNU_SOURCE -CFLAGS += -Wall -Werror -std=c11 -g +CFLAGS += -Wall -Werror -std=c11 CFLAGS += -Iinclude -CFLAGS += -lpthread -luuid +CFLAGS += -lpthread -luuid -lrt +CFLAGS += -fsanitize=address +CFLAGS += -fsanitize=undefined +CFLAGS += -g .PHONY: all clean dist install @@ -29,14 +32,17 @@ all: out/bin/sci out/obj/%.o: src/%.c | $(OBJDIR) $(CC) -c $? $(CFLAGS) -o $@ -OBJ += out/obj/main.o +OBJ += out/obj/api.o OBJ += out/obj/cli.o +OBJ += out/obj/executor.o OBJ += out/obj/log.o +OBJ += out/obj/main.o OBJ += out/obj/notify.o -OBJ += out/obj/util.o OBJ += out/obj/pipeline.o +OBJ += out/obj/strlist.o OBJ += out/obj/threadlist.o OBJ += out/obj/threadpool.o +OBJ += out/obj/util.o out/bin/sci: $(OBJ) | $(BINDIR) $(CC) -o $@ $^ $(CFLAGS) @@ -79,6 +85,9 @@ install: out/bin/sci mkdir -p $(DESTDIR)$(MANPREFIX)/man1 sed "s/VERSION/$(VERSION)/g" < src/sci.1 > $(DESTDIR)$(MANPREFIX)/man1/sci.1 chmod 644 $(DESTDIR)$(MANPREFIX)/man1/sci.1 + mkdir -p $(DESTDIR)$(MANPREFIX)/man7 + sed "s/VERSION/$(VERSION)/g" < src/sci-api.7 > $(DESTDIR)$(MANPREFIX)/man7/sci-api.7 + chmod 644 $(DESTDIR)$(MANPREFIX)/man7/sci-api.7 uninstall: # uninstall binaries @@ -87,3 +96,4 @@ uninstall: # uninstall services (only if system is using systemd though) # uninstall manpages rm -f $(DESTDIR)$(MANPREFIX)/man1/sci.1 + rm -f $(DESTDIR)$(MANPREFIX)/man7/sci-api.7 diff --git a/README.md b/README.md index 72fa79f..79f9065 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,9 @@ sudo make install PREFIX=/some/path ### Arch Linux It is recommended that you use an arch linux distribution when building. - +The `arch-builder.dockerfile` image provides a repeatable build environment. ```sh +docker build -t archbuilder -f arch-builder.dockerfile . ``` ### Debian @@ -65,7 +66,8 @@ docker build -t debbuilder deb-builder.dockerfile . ``` ## Brainstorm -If you dont want to congest your CI server. Too bad. Write faster ci suites. (TODO: implement runners) +If you dont want to congest your CI server, keep pipeline scripts fast or run them on separate machines. +Runner orchestration is intentionally outside the current core daemon. I would like to try to avoid writing a million REST APIs, as that just results in bloat usually. diff --git a/TODO.md b/TODO.md index 2395a32..dba99af 100644 --- a/TODO.md +++ b/TODO.md @@ -8,7 +8,7 @@ - [x] Fourth things fourth, implement a prototype that reads a space-separated file and populates a struct. - [x] Fifth things fifth, implement a prototype that spawns a new thread that executes a shell command. - [x] Sixth things sixth, daemonize it! - - [ ] Seventh things seventh, package the sucker (arch, debian, alpine, docker) + - [x] Seventh things seventh, package the sucker (arch, debian, alpine, docker) - [x] archlinux - https://wiki.archlinux.org/title/Creating_packages - [x] debian @@ -16,16 +16,22 @@ - just use docker. - [-] ~~alpine~~ later. - [-] ~~docker~~ later. - - [ ] Eight things eight, try it out! - maybe even write the python webhook extension. - - [ ] Ninth things ninth, fix bugs, see below + - [x] Eight things eight, try it out! - maybe even write the python webhook extension. + - [x] Port this document to gitea issue tracking + - [x] enable PATH-able programs and argv in the command section + - [x] custom environment variable passing. Something like `-e MY_TOKEN` ala docker-style + - [x] address sanitizers please. + - [ ] Ninth things ninth, fix bugs, see https://git.gtz.dk/agj/sci/projects/1 + - [ ] docstring in all header files - [ ] Tenth things tenth, write manpages, choose license - [ ] Eleventh things Eleventh, polish - [ ] Twelveth things last, release! - - Setup gitea.gtz.dk (will learn you how to set up subdomains (useful for shop.gtz.dk)) + - [x] Setup git.gtz.dk (will learn you how to set up subdomains (useful for shop.gtz.dk)) + - [ ] -1th things -1th, write a blog post about the tool (also set up your blog.gtz.dk) -BOOKMARK: okay. Now it feels like it's getting complicated. I want to run `sci` in a docker container. But that means +Okay. Now it feels like it's getting complicated. I want to run `sci` in a docker container. But that means that the build-threads also run in that docker container - meaning the container should have all the build dependencies -installed and we all know where that rabbithole goes. 9-30YB docker images with about a trillion unique build systems. +installed and we all know where that rabbithole goes. 9-30YiB docker images with about a trillion unique build systems. Let's not do that. The only alternative I can see is that the `sci` service is just not dockerized. The pipeline scripts can easily be dockerized themselves. Just have a `scripts/wget-src-dist-and-sci-sh-dockerized.sh` with `arg1` being the docker image @@ -33,12 +39,24 @@ to use? ```sh #!/bin/sh wget "$SCI_PIPELINE_URL" -docker run --rm -it --mount type=bind,source="$(pwd)"/thefileyouwgot.tar.gz,target=/thefileyouwgot.tar.gz,readonly --entrypoint sh $2 +docker run --rm -it -v .:/src -w /src $@ ``` Or something like that... Perhaps we can figure something out with an inline `ADD`, that also extracts the archive in the container or something. This approach is cleaner IMO. You can also more easily edit the `pipelines.conf` file if you need to. +The aforementioned rabbithole went like this: + - Let's say that `sci` is run inside a docker container. + This would make it very easy to deploy, but: + - Since pipelines are executed in the same environment as `sci`, either: + The `sci` container must be all-encompassing. i.e. it contains every single build system and scriptling language that + could possibly be used by any kind of user or; all pipelines must be run from a docker container themselves, meaning + that the `sci` container must have `dind`-privileges. Either option is suboptimal and will lock users into one way of + using `sci`, which is bad. + - Conclusion: Fuck docker. All environment management is delegated to the user and is not `sci`'s responsibility! + `sci` will always be run on the ci-machine itself, unless a user has provided a custom docker image, which is fine + and doesn't burden the `sci` project. + You were getting the following `pipelines.conf` file to work: ``` scih-dev ssh://git@git.gtz.dk:222/agj/scih.git scih-onpush /etc/sci/scripts/git-clone-and-run-sci-sh.sh @@ -54,25 +72,6 @@ alpine linux is using OpenRC (cool), which complicates things a little bit, but generally really well written. Otherwise, I am sure that both wiki.gentoo and wiki.archlinux have great pages too docker is super easy, just make a dockerfile - only concern is the trigger files. -#### Bugs / Missing Features - - [x] command output is being inherited. It should be piped into some random log-file - - [ ] pretty sure that `ctrl+c` / SIGINT is not graceful yet. - - [ ] missing license (heavily considering GPLv3) - - [ ] pipeline scripts should be executed in a unique `/tmp` dir - - [ ] Some way for third parties to see which pipelines are currently running and their status. - - Could be as simple as looking in the logs directory. - - How to mark a run as failed / success / warn? - - Third parties may need to extract artifacts. - or maybe the scripts themselves would upload artifacts? - - [ ] I am deliberately not using `Restart=on-failure` in the `scid.service` file because we are using `Type=exec` - and not `Type=notify` (yet) - which would require a `sd_notify` call of `READY=1` (see `man systemd.service`) - - [ ] Custom environment variables passed to the pipelines on invokation should be possible. - - [ ] Listener threads should be killed and restarted (worker pool should just chug along) when pipeline config file - has changed during runtime. Should be disableable with `--no-hot-reload-config` - i.e. on by default. - - [ ] `docker stop` is very slow. I am probably not handling signals properly yet. - - [x] It seems that `-v 4` is segfaulting when running release builds, maybe the logger just cant find the source file? - Nope. I just wrote some bad code (inverted NULL check). - ### Note Regarding `inotify` usage From the manpage: ``` diff --git a/include/api.h b/include/api.h new file mode 100644 index 0000000..6b75679 --- /dev/null +++ b/include/api.h @@ -0,0 +1,27 @@ +#ifndef SCI_API_H +#define SCI_API_H + +// Start the api. This will also trigger an "sci started" event. +// Note that this is a blocking call. +void* api_start(void*); + +// Fork the process and have the child run the api. +void api_start_p(); + +// Destroy all listeners and release the message queue. +void api_destroy(); + +// Post a newline-separated string with pipeline_id entries +// of the currently running pipelines on the message queue. +void api_list_running_pipelines(); + +// Trigger a pipeline started event. +void api_pipeline_started(const char* pipeline_id, const char* name); + +// Trigger a pipeline ended event. +void api_pipeline_ended(const char* pipeline_id, const char* name, int exit_code); + +// Trigger an api started event. +void api_started(); + +#endif // !SCI_API_H diff --git a/include/cli.h b/include/cli.h index 95cf127..348dda9 100644 --- a/include/cli.h +++ b/include/cli.h @@ -29,6 +29,8 @@ typedef struct { bool use_colors; optional_str log_file; optional_str pipeline_log_dir; + optional_str pipeline_cwd; + optional_strlist environment_vars; } cli_options; // Construct a new cli_options struct instance. diff --git a/include/executor.h b/include/executor.h new file mode 100644 index 0000000..c7066e2 --- /dev/null +++ b/include/executor.h @@ -0,0 +1,27 @@ +/** + * sci - a simple ci system + Copyright (C) 2024 Asger Gitz-Johansen + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ +#ifndef SCI_EXECUTOR_H +#define SCI_EXECUTOR_H +#include "strlist.h" + +void executor(void* pipeline_event); +void set_logdir(const char* logdir); +void set_working_directory(const char* cwd); +void set_shared_environment(const strlist_node* root); + +#endif diff --git a/include/notify.h b/include/notify.h index 205d6f3..2dc0fee 100644 --- a/include/notify.h +++ b/include/notify.h @@ -21,9 +21,12 @@ #include typedef void(*notify_callback)(pipeline_event* const); +typedef void(*config_change_callback)(); // Start listening for changes to the provided file. // Note that the `struct inotify_event*` provided is a managed pointer. void listen_for_changes(const pipeline_conf* config, notify_callback callback); +void listen_for_config_changes(const char* config_filepath, config_change_callback callback); + #endif diff --git a/include/optional.h b/include/optional.h index 77ebfc1..b1b2e1e 100644 --- a/include/optional.h +++ b/include/optional.h @@ -17,6 +17,7 @@ */ #ifndef SCI_OPTIONAL_H #define SCI_OPTIONAL_H +#include "strlist.h" #include #define optional_type(type) struct { bool has_value; type value; } @@ -24,5 +25,6 @@ typedef optional_type(int) optional_int; typedef optional_type(float) optional_float; typedef optional_type(char*) optional_str; typedef optional_type(const char*) optional_cstr; +typedef optional_type(strlist_node*) optional_strlist; #endif diff --git a/include/pipeline.h b/include/pipeline.h index 0830077..e0ebc9c 100644 --- a/include/pipeline.h +++ b/include/pipeline.h @@ -36,9 +36,13 @@ typedef struct { char* command; } pipeline_event; -// create a new pipeline_conf struct instance based on a configuration line. +// create a new `pipeline_conf` struct instance based on a configuration line. optional_pipeline_conf pipeline_create(const char* config_line); +void pipeline_event_destroy(pipeline_event* ev); +void pipeline_destroy(pipeline_conf* conf); void pipeline_register(pthread_t thread); void pipeline_loop(); +void pipeline_cancel(); +int pipeline_count(); #endif diff --git a/include/strlist.h b/include/strlist.h new file mode 100644 index 0000000..95f29d9 --- /dev/null +++ b/include/strlist.h @@ -0,0 +1,58 @@ +/** + * sci - a simple ci system + Copyright (C) 2024 Asger Gitz-Johansen + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ +#ifndef SCI_STRLIST_H +#define SCI_STRLIST_H + +// doubly linked list +typedef struct strlist_node { + char* str; + struct strlist_node* previous; + struct strlist_node* next; +} strlist_node; + +// Create a new root node. +// This function is not threadsafe. +strlist_node* create_strlist_node(char* str); + +// Add a new string to the string list. +// This function is not threadsafe. +strlist_node* add_str(char* str, strlist_node* root); + +// add a new string list node to the list. +// This function is not threadsafe. +strlist_node* add_str_node(strlist_node* str_node, strlist_node* root); + +// Remove a string list node from the list. +// This will free the str and stitch the "previous" and "next" ptrs. +// This function is not threadsafe. +void remove_strlist_node(strlist_node* node); + +// Completely clear the list. +// The list is completely invalid after this call and should be discarded. +// root itself will not be free'd by this function, but all content will be. +// This function is not threadsafe. +void clear_strlist(strlist_node* root); + +// Convert a strlist to an array of strings. +// Note that this copies the strlist, you still have to free it. +// The array itself is NULL terminated, enabling you to iterate to the end. +// The array should be free'd, as well as each of the entries. +// Returns NULL if the provided strlist is empty. +char** strlist_to_array(strlist_node* root); + +#endif diff --git a/include/threadlist.h b/include/threadlist.h index d7fb249..0590cfd 100644 --- a/include/threadlist.h +++ b/include/threadlist.h @@ -22,6 +22,7 @@ // doubly linked list implementation for managing threads typedef struct pthread_list_node { pthread_t thread; + pthread_mutex_t mutex; struct pthread_list_node* previous; struct pthread_list_node* next; } pthread_list_node; @@ -42,9 +43,8 @@ void remove_thread_node(pthread_list_node* node); // Completely clear the thread list. // This will call pthread_join on all nodes. // The list is completely invalid after this call and should be discarded. -// Note: -// - `root` has already been free'd. -// - this function is not thread-safe. +// Even root itself will be free'd by this function so it should be discarded as well. +// This function is not thread-safe. void clear_thread_list(pthread_list_node* root); #endif diff --git a/include/util.h b/include/util.h index 57095ab..1f054f5 100644 --- a/include/util.h +++ b/include/util.h @@ -40,5 +40,16 @@ typedef void(*line_handler)(const char*); void per_line(const char* file, line_handler handler); char* join(const char* a, const char* b); +char* join3(const char* a, const char* b, const char* c); +char* join4(const char* a, const char* b, const char* c, const char* d); +char* join5(const char* a, const char* b, const char* c, const char* d, const char* e); +char* join6(const char* a, const char* b, const char* c, const char* d, const char* e, const char* f); + +const char* skip_arg(const char* cp); +char* skip_spaces(const char* str); +int count_argc(const char* str); +char** argv_split(const char* str, int* argc_out); +void argv_free(char** argv); +int which(const char* program_name, char* out_full_program, int max_path); #endif diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..c381a22 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,3 @@ +# sci default scripts +This directory contains some default scripts that may or may not be useful to you. +Most of the scripts are fairly simple, but should be installed as part of the sci installation process. diff --git a/scripts/git-clone-and-run-sci-sh.sh b/scripts/git-clone-and-run-sci-sh.sh deleted file mode 100644 index 9d50675..0000000 --- a/scripts/git-clone-and-run-sci-sh.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -set -e -echo ">>> cloning..." -git clone $SCI_PIPELINE_URL $SCI_PIPELINE_NAME -cd $SCI_PIPELINE_NAME - -echo ">>> running .sci.sh..." -time sh .sci.sh diff --git a/scripts/git-clone-and-sci.sh b/scripts/git-clone-and-sci.sh new file mode 100755 index 0000000..ab08dcc --- /dev/null +++ b/scripts/git-clone-and-sci.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -ex # print all that we're doing (no need for echo's) +tmpdir=$(mktemp -d) +git clone --depth=1 --recurse-submodules --shallow-submodules -b $1 "$SCI_PIPELINE_URL" "$tmpdir" +shift +cd "$tmpdir" +sh .sci.sh +cd - diff --git a/scripts/wget-and-sci.sh b/scripts/wget-and-sci.sh new file mode 100755 index 0000000..5e7585e --- /dev/null +++ b/scripts/wget-and-sci.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# NOTE: This script assumes that the url is a .tar.gz file. +if [ "$#" -lt 1 ]; then + printf '%s\n' "usage: $0 EXTRACT_DIR" >&2 + printf '%s\n' "EXTRACT_DIR is the directory to enter after extracting the archive." >&2 + exit 2 +fi + +set -ex # print all that we're doing (no need for echo's) +env +tmpdir=$(mktemp -d) +wget "$SCI_PIPELINE_URL" -P "$tmpdir" +cd "$tmpdir" +tar xf *.tar.gz +cd "$1" +sh .sci.sh +cd - +rm -rf "$tmpdir" diff --git a/src/api.c b/src/api.c new file mode 100644 index 0000000..912c275 --- /dev/null +++ b/src/api.c @@ -0,0 +1,173 @@ +#include "api.h" +#include "log.h" +#include "util.h" +#include +#include +#include +#include +#include +#include + +#define LIST_REQ "list" +#define MQ_MAX_SIZE 8192 +#define SCI_API_EVENTS_QUEUE "/sci_events" +#define SCI_API_REQUESTS_QUEUE "/sci_requests" + +bool api_is_running = false; +mqd_t api_out = (mqd_t)-1; +mqd_t api_in = (mqd_t)-1; + +typedef struct api_pipeline_node { + char* pipeline_id; + char* name; + struct api_pipeline_node* next; +} api_pipeline_node; + +pthread_mutex_t api_pipeline_mutex = PTHREAD_MUTEX_INITIALIZER; +api_pipeline_node* api_running_pipelines = NULL; + +static void api_send(const char* msg) { + if(mq_send(api_out, msg, strnlen(msg, MQ_MAX_SIZE), 1) == -1) + perror("mq_send"); +} + +static void api_register_pipeline(const char* pipeline_id, const char* name) { + api_pipeline_node* node = malloc(sizeof(api_pipeline_node)); + node->pipeline_id = strdup(pipeline_id); + node->name = strdup(name); + + pthread_mutex_lock(&api_pipeline_mutex); + node->next = api_running_pipelines; + api_running_pipelines = node; + pthread_mutex_unlock(&api_pipeline_mutex); +} + +static void api_unregister_pipeline(const char* pipeline_id) { + pthread_mutex_lock(&api_pipeline_mutex); + api_pipeline_node* previous = NULL; + api_pipeline_node* cursor = api_running_pipelines; + while(cursor != NULL) { + if(strncmp(cursor->pipeline_id, pipeline_id, MQ_MAX_SIZE) == 0) { + if(previous == NULL) + api_running_pipelines = cursor->next; + else + previous->next = cursor->next; + free(cursor->pipeline_id); + free(cursor->name); + free(cursor); + pthread_mutex_unlock(&api_pipeline_mutex); + return; + } + previous = cursor; + cursor = cursor->next; + } + pthread_mutex_unlock(&api_pipeline_mutex); +} + +void api_handle_request(const char* request) { + log_trace("api request: '%s'", request); + + // list + if(strncmp(request, LIST_REQ, MQ_MAX_SIZE) == 0) { + api_list_running_pipelines(); + return; + } + + // else + log_error("unrecognized api request: '%s'", request); +} + +void* api_start(void* data) { + struct mq_attr attr; + attr.mq_flags = 0; + attr.mq_maxmsg = 10; + attr.mq_msgsize = MQ_MAX_SIZE; + attr.mq_curmsgs = 0; + + api_out = mq_open(SCI_API_EVENTS_QUEUE, O_CREAT | O_WRONLY, 0666, &attr); + if(api_out == -1) { + perror("mq_open"); + return NULL; + } + + api_in = mq_open(SCI_API_REQUESTS_QUEUE, O_CREAT | O_RDONLY, 0666, &attr); + if(api_in == -1) { + perror("mq_open"); + return NULL; + } + + api_started(); + + log_info("api listening for requests"); + char msg[MQ_MAX_SIZE]; + api_is_running = true; + while(api_is_running) { + memset(msg, '\0', MQ_MAX_SIZE); + if(mq_receive(api_in, msg, MQ_MAX_SIZE, NULL) == -1) { + perror("mq_receive"); + return NULL; + } + api_handle_request(msg); + } + return NULL; +} + +void api_start_p() { + pthread_t conf_listener; + ASSERT_SYSCALL_SUCCESS(pthread_create(&conf_listener, NULL, &api_start, (void*)NULL)); + pthread_setname_np(conf_listener, "sci-api"); +} + +void api_destroy() { + log_trace("closing api"); + api_is_running = false; + if(api_in != (mqd_t)-1) + mq_close(api_in); + if(api_out != (mqd_t)-1) + mq_close(api_out); + mq_unlink(SCI_API_REQUESTS_QUEUE); + mq_unlink(SCI_API_EVENTS_QUEUE); +} + +void api_list_running_pipelines() { + char msg[MQ_MAX_SIZE]; + msg[0] = '\0'; + size_t used = 0; + + pthread_mutex_lock(&api_pipeline_mutex); + api_pipeline_node* cursor = api_running_pipelines; + while(cursor != NULL) { + int written = snprintf(msg + used, MQ_MAX_SIZE - used, "%s%s", used == 0 ? "" : "\n", cursor->pipeline_id); + if(written < 0) + break; + if((size_t)written >= MQ_MAX_SIZE - used) { + log_error("running pipeline list exceeds api message size"); + msg[MQ_MAX_SIZE - 1] = '\0'; + break; + } + used += written; + cursor = cursor->next; + } + pthread_mutex_unlock(&api_pipeline_mutex); + api_send(msg); +} + +void api_pipeline_started(const char* pipeline_id, const char* name) { + api_register_pipeline(pipeline_id, name); + char* msg = join("pipeline_new ", pipeline_id); + api_send(msg); + free(msg); +} + +void api_pipeline_ended(const char* pipeline_id, const char* name, int exit_code) { + api_unregister_pipeline(pipeline_id); + char exit_code_str[64]; + snprintf(exit_code_str, sizeof(exit_code_str), "%d", exit_code); + char* msg = join6("pipeline_end ", pipeline_id, " ", name, " ", exit_code_str); + api_send(msg); + free(msg); +} + +void api_started() { + api_send("sci_started"); +} diff --git a/src/cli.c b/src/cli.c index c8b8aa3..fe4e010 100644 --- a/src/cli.c +++ b/src/cli.c @@ -15,11 +15,11 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ +#include "cli.h" +#include #include #include #include -#include -#include "cli.h" cli_options new_options() { cli_options result; @@ -50,6 +50,23 @@ cli_options new_options() { char* pipeline_log_dir = getenv("SCI_PIPELINE_LOG_DIR"); result.pipeline_log_dir.has_value = pipeline_log_dir != NULL; result.pipeline_log_dir.value = pipeline_log_dir; + + char* pipeline_cwd = getenv("SCI_PIPELINE_CWD"); + result.pipeline_cwd.has_value = pipeline_cwd != NULL; + result.pipeline_cwd.value = pipeline_cwd; + + char* environment_vars = getenv("SCI_PIPELINE_ENV_VARS"); + if(environment_vars == NULL) { + result.environment_vars.has_value = false; + result.environment_vars.value = NULL; + } else { + char* tok = strtok(environment_vars, ";"); + result.environment_vars.has_value = true; + result.environment_vars.value = create_strlist_node(tok); + tok = strtok(NULL, ";"); + while(tok != NULL) + add_str(tok, result.environment_vars.value); + } return result; } @@ -60,24 +77,28 @@ void destroy_options(cli_options v) { free(v.log_file.value); if(v.pipeline_log_dir.has_value) free(v.pipeline_log_dir.value); + if(v.environment_vars.has_value) + clear_strlist(v.environment_vars.value); } // . + */ +#include "executor.h" +#include "api.h" +#include "log.h" +#include "optional.h" +#include "pipeline.h" +#include "strlist.h" +#include "util.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +const char* log_dir = "."; +const char* cwd = "/tmp"; +const strlist_node* shared_environment = NULL; + +void set_shared_environment(const strlist_node* root) { + shared_environment = root; +} + +void set_logdir(const char* logdir) { + log_dir = logdir; + struct stat st = {0}; + if(stat(log_dir, &st) == -1) + mkdir(log_dir, 0700); +} + +void set_working_directory(const char* _cwd) { + cwd = _cwd; + struct stat st = {0}; + if(stat(cwd, &st) == -1) + mkdir(cwd, 0700); +} + +char* create_pipeline_id() { + uuid_t uuid; + uuid_generate(uuid); + // example uuid + // 662ddee9-ee7c-4d13-8999-a2604c6d12d6 + // it's 36 characters (+null) + char* pipeline_id = malloc(sizeof(char) * 37); + uuid_unparse_lower(uuid, pipeline_id); + return pipeline_id; +} + +optional_int open_logfile(const char* const pipeline_id) { + optional_int result; + result.has_value = false; + result.value = 0; + char* log_filepath = join4(log_dir, "/", pipeline_id, ".log"); + int fd = open(log_filepath, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd != -1) { + result.has_value = true; + result.value = fd; + } else + perror("open"); + free(log_filepath); + return result; +} + +void add_joined_str(strlist_node* root, const char* a, const char* b) { + char* tmp = join(a, b); + add_str(tmp, root); + free(tmp); +} + +void add_env(strlist_node* root, const char* env) { + char* tmp = join3(env, "=", getenv(env)); + add_str(tmp, root); + free(tmp); +} + +char** create_environment(const pipeline_event* const e, const char* pipeline_id) { + char* tmp = join("SCI_PIPELINE_NAME=", e->name); + strlist_node* env = create_strlist_node(tmp); + free(tmp); + add_joined_str(env, "SCI_PIPELINE_URL=", e->url); + add_joined_str(env, "SCI_PIPELINE_TRIGGER=", e->trigger); + add_joined_str(env, "SCI_PIPELINE_ID=", pipeline_id); + if(shared_environment != NULL) { + const strlist_node* cursor = shared_environment; + while(cursor != NULL) { + add_env(env, cursor->str); + cursor = cursor->next; + } + } + char** envp = strlist_to_array(env); + clear_strlist(env); + return envp; +} + +void executor(void* data) { + // Create pipeline id + char* pipeline_id = create_pipeline_id(); + + // Create logfile path + optional_int fd = open_logfile(pipeline_id); + if(!fd.has_value) { + log_error("could not open log file - not starting pipeline"); + return; + } + + // spawn the process + pid_t pid; + const pipeline_event* const e = data; + char** envp = create_environment(e, pipeline_id); + int argc; + char** argv = argv_split(e->command, &argc); + log_trace("executing pipeline %s with argv:", e->name); + for(int i = 0; i < argc; i++) + log_trace(" \"%s\"", argv[i]); + char arg0[PATH_MAX]; + if(which(argv[0], arg0, PATH_MAX) == -1) + goto end; + + // fork / cwd / exec idiom + pid = fork(); + if(pid < 0) { + perror("fork"); + goto end; // I know. The raptors have picked up the scent. I'll just have to mask it with more stinky code. + } + + if(pid == 0) { + // child process + dup2(fd.value, STDOUT_FILENO); + dup2(fd.value, STDERR_FILENO); + char* pipeline_cwd = join3(cwd, "/", pipeline_id); + struct stat st = {0}; + if(stat(pipeline_cwd, &st) == -1) + mkdir(pipeline_cwd, 0700); + chdir(pipeline_cwd); + free(pipeline_cwd); + execvpe(arg0, argv, envp); + return; + } + + api_pipeline_started(pipeline_id, e->name); + log_info("{%s} (%s) spawned", pipeline_id, e->name); + + // Wait for process to complete + int status; + waitpid(pid, &status, 0); + api_pipeline_ended(pipeline_id, e->name, status); + log_info("{%s} (%s) [pid=%d] exited with status %d", pipeline_id, e->name, pid, status); + char buf[32]; + sprintf(buf, "exited with status %d", status); + if(write(fd.value, buf, strnlen(buf, 32)) == -1) + perror("write"); +end: + argv_free(argv); + close(fd.value); + free(pipeline_id); + pipeline_event_destroy(data); + char** cursor = envp; + while(*cursor != NULL) { + free(*cursor); + cursor++; + } + free(envp); +} diff --git a/src/log.c b/src/log.c index e48c3a9..0a8145a 100644 --- a/src/log.c +++ b/src/log.c @@ -78,6 +78,7 @@ void log_log(const char* file, int line, int level, const char* fmt, ...) { g_log_tm = localtime(&t); } strftime(timestamp, sizeof(timestamp), "%H:%M:%S", g_log_tm); + g_log_tm = NULL; const char* level_color = get_level_color(level); const char* level_name = get_level_name(level); diff --git a/src/main.c b/src/main.c index a68048b..6eabfb5 100644 --- a/src/main.c +++ b/src/main.c @@ -15,104 +15,41 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ +#include "api.h" #include "cli.h" +#include "executor.h" #include "log.h" #include "notify.h" #include "pipeline.h" #include "threadpool.h" #include "util.h" -#include -#include +#include +#include +#include #include #include -#include -#include +#include -threadpool* pool = NULL; -char* log_dir = "./"; // NOTE: must end with a / - -char* create_pipeline_id() { - uuid_t uuid; - uuid_generate(uuid); - char* pipeline_id = malloc(32); - uuid_unparse_lower(uuid, pipeline_id); - return pipeline_id; -} - -optional_int open_logfile(const char* const pipeline_id) { - optional_int result; - result.has_value = false; - result.value = 0; - char* log_file = join(pipeline_id, ".log"); - char* log_filepath = join(log_dir, log_file); - int fd = open(log_filepath, O_WRONLY | O_CREAT | O_TRUNC, 0644); - if (fd != -1) { - result.has_value = true; - result.value = fd; - } else - perror("open"); - free(log_file); - free(log_filepath); - return result; -} - -void executor(void* data) { - // Create pipeline id - char* pipeline_id = create_pipeline_id(); - - // Create logfile path - optional_int fd = open_logfile(pipeline_id); - if(!fd.has_value) - return; - - // spawn the process - pid_t pid; - posix_spawn_file_actions_t actions; - posix_spawn_file_actions_init(&actions); - posix_spawn_file_actions_adddup2(&actions, fd.value, STDOUT_FILENO); - posix_spawn_file_actions_adddup2(&actions, fd.value, STDERR_FILENO); - const pipeline_event* const e = data; - char* name = join("SCI_PIPELINE_NAME=", e->name); - char* url = join("SCI_PIPELINE_URL=", e->url); - char* trigger = join("SCI_PIPELINE_TRIGGER=", e->trigger); - char* id = join("SCI_PIPELINE_ID=", pipeline_id); - char* envp[] = { name, url, trigger, id, NULL }; - char* argv[] = { "/bin/sh", "-c", e->command, NULL }; - if(posix_spawn(&pid, "/bin/sh", &actions, NULL, argv, envp) != 0) { - perror("posix_spawn"); - goto end; // I know. The raptors have picked up the scent. I'll just have to mask it with more poopy code. - } - log_trace("{%s} (%s) spawned", pipeline_id, e->name); - - // Wait for process to complete - int status; - waitpid(pid, &status, 0); - if(WIFEXITED(status)) - log_trace("{%s} (%s) exited with status %d", pipeline_id, e->name, WEXITSTATUS(status)); -end: - close(fd.value); - free(pipeline_id); - free(name); - free(url); - free(trigger); - free(id); -} +threadpool* worker_pool = NULL; +char* trigger_dir = "/tmp/sci"; +bool config_file_changed = false; void on_event(pipeline_event* const e) { - if(!threadpool_add_work(pool, executor, (void*)e)) + if(!threadpool_add_work(worker_pool, executor, (void*)e)) log_error("could not add work to the threadpool"); } -void* listen_for_changes_thread(void* data) { - const pipeline_conf* conf = (const pipeline_conf*)data; - while(1) // TODO: Should be while(sigint_has_not_been_caught) instead - listen_for_changes(conf, &on_event); +void listener_thread_cleanup(void* data) { // We're now done with the config. - free(conf->name); - free(conf->url); - free(conf->trigger); - free(conf->command); - free(data); + pipeline_destroy((pipeline_conf*)data); +} + +void* listen_for_changes_thread(void* data) { + pipeline_conf* conf = (pipeline_conf*)data; + pthread_cleanup_push(listener_thread_cleanup, conf); + while(1) + listen_for_changes(conf, &on_event); + pthread_cleanup_pop(1); return NULL; } @@ -134,18 +71,31 @@ void config_interpret_line(const char* line) { log_error("unable to register pipeline"); return; } - char* dest; - // NOTE: trigger names are allowed max 32 characters - dest = malloc(sizeof(*dest) * (9+33)); - dest[0] = '\0'; - strncat(dest, "/tmp/sci/", 10); - strncat(dest, conf.value->trigger, 33); + char* new_trigger_val = join3(trigger_dir, "/", conf.value->trigger); free(conf.value->trigger); - conf.value->trigger = dest; + conf.value->trigger = new_trigger_val; pthread_t t = spawn_listener(conf.value); pipeline_register(t); } +void on_config_file_changed() { + config_file_changed = true; + pipeline_cancel(); + log_info("config file changed, reloading..."); +} + +void* listen_for_config_changes_thread(void* data) { + while(1) + listen_for_config_changes((const char*)data, &on_config_file_changed); + return NULL; +} + +void signal_handler(int signal) { + log_info("signal retrieved"); + if(signal == SIGINT) + pipeline_cancel(); +} + int main(int argc, char** argv) { cli_options args = parse(argc, argv); log_settings settings; @@ -154,6 +104,8 @@ int main(int argc, char** argv) { settings.out_file = args.log_file.has_value ? fopen(args.log_file.value, "w+") : stdout; log_init(settings); + signal(SIGINT, signal_handler); + if(args.help) { print_help(stdout, argv[0]); exit(EXIT_SUCCESS); @@ -169,24 +121,39 @@ int main(int argc, char** argv) { exit(EXIT_FAILURE); } - if(args.pipeline_log_dir.has_value) - log_dir = args.pipeline_log_dir.value; - - struct stat st = {0}; - if(stat(log_dir, &st) == -1) - mkdir(log_dir, 0700); - - if(stat("/tmp/sci", &st) == -1) - mkdir("/tmp/sci", 0700); - if(access(args.config_file.value, F_OK) != 0) { fprintf(stderr, "no such file or directory %s\n", args.config_file.value); exit(EXIT_FAILURE); } - - pool = threadpool_create(args.executors); - per_line(args.config_file.value, &config_interpret_line); - pipeline_loop(); - threadpool_destroy(pool); + if(args.pipeline_log_dir.has_value) + set_logdir(args.pipeline_log_dir.value); + + if(args.pipeline_cwd.has_value) + set_working_directory(args.pipeline_cwd.value); + + struct stat st = {0}; + if(stat(trigger_dir, &st) == -1) + mkdir(trigger_dir, 0700); + + if(args.environment_vars.has_value) + set_shared_environment(args.environment_vars.value); + + api_start_p(); + + log_info("spawning trigger thread for config file"); + pthread_t conf_listener; + ASSERT_SYSCALL_SUCCESS(pthread_create(&conf_listener, NULL, &listen_for_config_changes_thread, (void*)args.config_file.value)); + pthread_setname_np(conf_listener, "sci-conf-listener"); + do { + config_file_changed = false; + worker_pool = threadpool_create(args.executors); + per_line(args.config_file.value, &config_interpret_line); + log_info("listening for pipeline invocations"); + pipeline_loop(); + } while(config_file_changed); + pthread_cancel(conf_listener); + threadpool_destroy(worker_pool); + destroy_options(args); + api_destroy(); } diff --git a/src/notify.c b/src/notify.c index 753ea7a..80d53a8 100644 --- a/src/notify.c +++ b/src/notify.c @@ -18,12 +18,28 @@ #include "notify.h" #include "util.h" #include "log.h" +#include +#include +#include #define EV_SIZE sizeof(struct inotify_event) #define BUF_LEN EV_SIZE * 32 +static void dispatch_pipeline_events(const pipeline_conf* config, notify_callback callback, char* buffer, ssize_t len) { + for(int i = 0; i < len; ) { + struct inotify_event* e = (struct inotify_event*)&buffer[i]; + pipeline_event* ev = malloc(sizeof(pipeline_event)); + ev->event = e; + ev->name = strdup(config->name); + ev->url = strdup(config->url); + ev->trigger = strdup(config->trigger); + ev->command = strdup(config->command); + callback(ev); + i += EV_SIZE + e->len; + } +} + void listen_for_changes(const pipeline_conf* config, notify_callback callback) { - // TODO: callback is a bit slow sometimes. We should also poll once after calling callback const char* filename = config->trigger; if(access(filename, F_OK) != 0) { log_trace("file does not exist yet, creating it."); @@ -32,21 +48,46 @@ void listen_for_changes(const pipeline_conf* config, notify_callback callback) { } int fd = inotify_init(); ASSERT_SYSCALL_SUCCESS(fd); - inotify_add_watch(fd, filename, IN_ATTRIB); + ASSERT_SYSCALL_SUCCESS(inotify_add_watch(fd, filename, IN_ATTRIB)); log_trace("listening for changes in file: %s", filename); char buffer[BUF_LEN]; - int r = read(fd, buffer, BUF_LEN); + ssize_t r = read(fd, buffer, BUF_LEN); + if(r == -1) { + perror("read"); + return; + } + dispatch_pipeline_events(config, callback, buffer, r); + + int flags = fcntl(fd, F_GETFL, 0); + if(flags != -1 && fcntl(fd, F_SETFL, flags | O_NONBLOCK) != -1) { + while((r = read(fd, buffer, BUF_LEN)) > 0) + dispatch_pipeline_events(config, callback, buffer, r); + if(r == -1 && errno != EAGAIN && errno != EWOULDBLOCK) + perror("read"); + } + + ASSERT_SYSCALL_SUCCESS(close(fd)); +} + +void listen_for_config_changes(const char* config_filepath, config_change_callback callback) { + if(access(config_filepath, F_OK) != 0) { + perror("access"); + return; + } + int fd = inotify_init(); + ASSERT_SYSCALL_SUCCESS(fd); + ASSERT_SYSCALL_SUCCESS(inotify_add_watch(fd, config_filepath, IN_ATTRIB)); + log_trace("listening for changes in file: %s", config_filepath); + char buffer[BUF_LEN]; + ssize_t r = read(fd, buffer, BUF_LEN); + if(r == -1) { + perror("read"); + return; + } assert(r != -1); for(int i = 0; i < r; ) { - struct inotify_event* e = (struct inotify_event*)&buffer[i]; - pipeline_event ev; - ev.event = e; - ev.name = config->name; - ev.url = config->url; - ev.trigger = config->trigger; - ev.command = config->command; - callback(&ev); - i += EV_SIZE + e->len; + callback(); + i += EV_SIZE + ((struct inotify_event*)&buffer[i])->len;; } ASSERT_SYSCALL_SUCCESS(close(fd)); } diff --git a/src/pipeline.c b/src/pipeline.c index cc2feef..824127d 100644 --- a/src/pipeline.c +++ b/src/pipeline.c @@ -19,12 +19,14 @@ #include "pipeline.h" #include "threadlist.h" #include "util.h" +#include #include #include pthread_list_node* root = NULL; optional_pipeline_conf pipeline_create(const char* config_line) { + log_trace("pipeline create"); optional_pipeline_conf result; result.has_value = false; const char* pattern = "[^[:blank:]]+|\"[^\"]*\""; @@ -40,15 +42,21 @@ optional_pipeline_conf pipeline_create(const char* config_line) { break; off = pmatch[0].rm_so + (cursor - config_line); len = pmatch[0].rm_eo - pmatch[0].rm_so; - opts[i] = strndup(config_line + off, len); + // Cut off the "-s if it is string-enclosed + if(config_line[off] == '"' && config_line[off+len-1] == '"') + opts[i] = strndup(config_line + off+1, len-2); + else + opts[i] = strndup(config_line + off, len); cursor += pmatch[0].rm_eo; } if(i != 4) { - log_error("invalid configuration!\nline is invalid: \"%s\""); + log_error("invalid configuration!"); + log_error("line is invalid: \"%s\"", config_line); for(int j = i-1; j >= 0; j--) free(opts[j]); return result; } + regfree(®); result.value = malloc(sizeof(pipeline_conf)); result.value->name = opts[0]; @@ -64,7 +72,17 @@ optional_pipeline_conf pipeline_create(const char* config_line) { return result; } +void pipeline_destroy(pipeline_conf* conf) { + log_trace("pipeline destroy"); + free(conf->name); + free(conf->url); + free(conf->trigger); + free(conf->command); + free(conf); +} + void pipeline_register(pthread_t thread) { + log_trace("pipeline register thread"); if(root == NULL) { root = create_thread_node(thread); return; @@ -73,6 +91,35 @@ void pipeline_register(pthread_t thread) { } void pipeline_loop() { + log_trace("pipeline loop"); clear_thread_list(root); root = NULL; } + +void pipeline_cancel() { + log_trace("cancelling pipeline"); + pthread_list_node* cursor = root; + while(cursor != NULL) { + pthread_cancel(cursor->thread); + cursor = cursor->next; + } +} + +void pipeline_event_destroy(pipeline_event* ev) { + log_trace("pipeline event destroy"); + free(ev->name); + free(ev->trigger); + free(ev->url); + free(ev->command); + free(ev); +} + +int pipeline_count() { + int result = 0; + pthread_list_node* cursor = root; + while(cursor != NULL) { + cursor = cursor->next; + result++; + } + return result; +} diff --git a/src/sci-api.7 b/src/sci-api.7 new file mode 100644 index 0000000..3b78f71 --- /dev/null +++ b/src/sci-api.7 @@ -0,0 +1,35 @@ +.TH sci-api 7 2024-08-17 "VERSION" "Simple CI manual" + +.SH NAME +sci-api - POSIX message queue interface for sci + +.SH DESCRIPTION +.B sci +exposes a small local API through POSIX message queues. +Clients send requests to +.I /sci_requests +and receive events and responses from +.I /sci_events. + +.SH REQUESTS +.TP +.B list +Return the currently running pipeline IDs as a newline-separated message. +An empty message means no pipelines are running. + +.SH EVENTS +.TP +.B sci_started +The daemon API thread has started. +.TP +.BI pipeline_new " pipeline_id" +A pipeline process has started. +.TP +.BI pipeline_end " pipeline_id name exit_status" +A pipeline process has exited. + +.SH LIMITS +Messages are limited to 8192 bytes. + +.SH SEE ALSO +.BR sci (1) diff --git a/src/sci.1 b/src/sci.1 index 6e69f4a..83094f6 100644 --- a/src/sci.1 +++ b/src/sci.1 @@ -42,14 +42,12 @@ The operation of is configured through a .I pipelines.conf configuration file (see -.I sci(7) +.BR sci (5) for configuration language details) and each pipeline will have an associated pipeline trigger file that can be By default, pipeline triggers are placed in /tmp/sci but this can be overridden with the .OP -x. - - .SH EXAMPLES A simple example configuration file could look something like the following: @@ -71,7 +69,6 @@ disable all warnings. .SH AUTHOR Asger Gitz\-Johansen . -\" TODO: decide on license .SH COPYRIGHT Copyright (C) 2024 Asger Gitz-Johansen @@ -88,5 +85,6 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . -.SH SEE ALSO -\" TODO: write sci(7) +.SH "SEE ALSO" +.BR sci (5), +.BR sci-api (7), diff --git a/src/strlist.c b/src/strlist.c new file mode 100644 index 0000000..3f26f57 --- /dev/null +++ b/src/strlist.c @@ -0,0 +1,91 @@ +#include "strlist.h" +#include +#include + +#define MAX_STRLEN 512 + +strlist_node* create_strlist_node(char* str) { + strlist_node* new = malloc(sizeof(strlist_node)); + new->previous = NULL; + new->next = NULL; + if(str) + new->str = strndup(str, MAX_STRLEN); + else + new->str = NULL; + return new; +} + +strlist_node* add_str(char* str, strlist_node* root) { + strlist_node* cursor = root; + while(cursor->next != NULL) + cursor = cursor->next; + strlist_node* new = malloc(sizeof(strlist_node)); + new->previous = cursor; + new->next = NULL; + new->str = strndup(str, MAX_STRLEN); + cursor->next = new; + return new; +} + +strlist_node* add_str_node(strlist_node* root, strlist_node* node) { + strlist_node* cursor = root; + while(cursor->next != NULL) + cursor = cursor->next; + node->previous = cursor; + node->next = NULL; + cursor->next = node; + return node; +} + +void remove_strlist_node(strlist_node* node) { + strlist_node* prev = node->previous; + strlist_node* next = node->next; + if(node->str) + free(node->str); + node->str = NULL; + free(node); + if(prev != NULL) + prev->next = next; + if(next != NULL) + next->previous = prev; +} + +void clear_strlist(strlist_node* root) { + strlist_node* cursor = root; + while(cursor != NULL) { + if(cursor->str != NULL) + free(cursor->str); + strlist_node* prev = cursor; + cursor = cursor->next; + free(prev); + } +} + +size_t strlist_length(strlist_node* root) { + size_t result = 0; + strlist_node* cursor = root; + while(cursor != NULL) { + result++; + cursor = cursor->next; + } + return result; +} + +char** strlist_to_array(strlist_node* root) { + size_t len = strlist_length(root); + if(len <= 0) + return NULL; + char** result = malloc(sizeof(char*) * (len + 1)); + for(int i = 0; i < len+1; i++) + result[i] = NULL; + strlist_node* cursor = root; + for(int i = 0; i < len; i++) { + if(cursor == NULL) + break; + if(cursor->str == NULL) + continue; + result[i] = strdup(cursor->str); + cursor = cursor->next; + } + return result; +} diff --git a/src/threadlist.c b/src/threadlist.c index a3de7e7..f76970e 100644 --- a/src/threadlist.c +++ b/src/threadlist.c @@ -18,19 +18,23 @@ #include "threadlist.h" #include -// TODO: mutex should really be per-list. -pthread_mutex_t threadlist_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_list_node* threadlist_root(pthread_list_node* node) { + while(node != NULL && node->previous != NULL) + node = node->previous; + return node; +} pthread_list_node* create_thread_node(pthread_t thread) { pthread_list_node* new = malloc(sizeof(pthread_list_node)); new->previous = NULL; new->next = NULL; new->thread = thread; + pthread_mutex_init(&new->mutex, NULL); return new; } pthread_list_node* add_thread(pthread_t thread, pthread_list_node* root) { - pthread_mutex_lock(&threadlist_mutex); + pthread_mutex_lock(&root->mutex); pthread_list_node* cursor = root; while(cursor->next != NULL) cursor = cursor->next; @@ -38,40 +42,49 @@ pthread_list_node* add_thread(pthread_t thread, pthread_list_node* root) { new->previous = cursor; new->next = NULL; new->thread = thread; + pthread_mutex_init(&new->mutex, NULL); cursor->next = new; - pthread_mutex_unlock(&threadlist_mutex); + pthread_mutex_unlock(&root->mutex); return new; } pthread_list_node* add_thread_node(pthread_list_node* root, pthread_list_node* node) { - pthread_mutex_lock(&threadlist_mutex); + pthread_mutex_lock(&root->mutex); pthread_list_node* cursor = root; while(cursor->next != NULL) cursor = cursor->next; node->previous = cursor; node->next = NULL; cursor->next = node; - pthread_mutex_unlock(&threadlist_mutex); + pthread_mutex_unlock(&root->mutex); return node; } void remove_thread_node(pthread_list_node* node) { - pthread_mutex_lock(&threadlist_mutex); + if(node == NULL) + return; + pthread_list_node* root = threadlist_root(node); + pthread_mutex_lock(&root->mutex); pthread_list_node* prev = node->previous; pthread_list_node* next = node->next; - free(node); if(prev != NULL) prev->next = next; if(next != NULL) next->previous = prev; - pthread_mutex_unlock(&threadlist_mutex); + pthread_mutex_unlock(&root->mutex); + pthread_mutex_destroy(&node->mutex); + free(node); } void clear_thread_list(pthread_list_node* root) { + if(root == NULL) + return; pthread_list_node* cursor = root; while(cursor != NULL) { pthread_join(cursor->thread, NULL); + pthread_list_node* prev = cursor; cursor = cursor->next; - free(cursor->previous); + pthread_mutex_destroy(&prev->mutex); + free(prev); } } diff --git a/src/threadpool.c b/src/threadpool.c index 0cf9aad..8f19191 100644 --- a/src/threadpool.c +++ b/src/threadpool.c @@ -41,28 +41,33 @@ static void threadpool_work_destroy(threadpool_work* work) { free(work); } -static threadpool_work* threadpool_work_pop(threadpool* pool) { - threadpool_work* work; +static optional_threadpool_work threadpool_work_pop(threadpool* pool) { + optional_threadpool_work result; + result.value = NULL; + result.has_value = false; if (pool == NULL) - return NULL; + return result; - work = pool->work_first; + threadpool_work* work = pool->work_first; if (work == NULL) - return NULL; // TODO: This should propbably be using optionals + return result; if (work->next == NULL) { pool->work_first = NULL; pool->work_last = NULL; - return work; + result.value = work; + result.has_value = true; + return result; } pool->work_first = work->next; - return work; + result.value = work; + result.has_value = true; + return result; } static void* threadpool_worker(void* arg) { log_trace("threadpool worker spawned"); threadpool* pool = arg; - threadpool_work* work; while(true) { // Wait for work pthread_mutex_lock(&(pool->work_mutex)); @@ -70,18 +75,20 @@ static void* threadpool_worker(void* arg) { pthread_cond_wait(&(pool->work_cond), &(pool->work_mutex)); if (pool->stop) break; - work = threadpool_work_pop(pool); - pool->working_count++; + optional_threadpool_work work = threadpool_work_pop(pool); + if(work.has_value) + pool->working_count++; pthread_mutex_unlock(&(pool->work_mutex)); // Do the work - if (work != NULL) { - work->func(work->arg); - threadpool_work_destroy(work); + if (work.has_value) { + work.value->func(work.value->arg); + threadpool_work_destroy(work.value); } pthread_mutex_lock(&(pool->work_mutex)); - pool->working_count--; + if(work.has_value) + pool->working_count--; if (!pool->stop && pool->working_count == 0 && pool->work_first == NULL) pthread_cond_signal(&(pool->working_cond)); pthread_mutex_unlock(&(pool->work_mutex)); diff --git a/src/util.c b/src/util.c index e49ac71..9c8e147 100644 --- a/src/util.c +++ b/src/util.c @@ -15,49 +15,180 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -#include "util.h" #include "log.h" +#include "util.h" #include +#include #include char* trim(const char* const str) { - char* begin = strdup(str); - char* end; - while(isspace((unsigned char)*begin)) - begin++; - if(*begin == 0) - return begin; - end = begin + strlen(begin) - 1; - while(end > begin && isspace((unsigned char)*end)) - end--; - *(end + 1) = '\0'; - return begin; + char* begin = strdup(str); + char* end; + while(isspace((unsigned char)*begin)) + begin++; + if(*begin == 0) + return begin; + end = begin + strlen(begin) - 1; + while(end > begin && isspace((unsigned char)*end)) + end--; + *(end + 1) = '\0'; + return begin; } void per_line(const char* file, line_handler handler) { - FILE* stream; - char* line = NULL; - size_t len = 0; - ssize_t nread; - log_trace("reading file %s", file); - stream = fopen(file, "r"); - if(stream == NULL) { - perror("fopen"); - return; - } - while((nread = getline(&line, &len, stream)) != -1) { - char* line_trimmed = trim(line); - handler(line_trimmed); - free(line_trimmed); - } - free(line); - fclose(stream); + FILE* stream; + char* line = NULL; + size_t len = 0; + ssize_t nread; + log_trace("reading file %s", file); + stream = fopen(file, "r"); + if(stream == NULL) { + perror("fopen"); + return; + } + while((nread = getline(&line, &len, stream)) != -1) { + char* line_trimmed = trim(line); + handler(line_trimmed); + free(line_trimmed); + } + free(line); + fclose(stream); } char* join(const char* a, const char* b) { - size_t alen = strlen(a); - size_t blen = strlen(b); - char* result = malloc(alen + blen + 1); - sprintf(result, "%s%s", a, b); - return result; + size_t alen = strlen(a); + size_t blen = strlen(b); + char* result = malloc(alen + blen + 1); + sprintf(result, "%s%s", a, b); + return result; +} + +char* join3(const char* a, const char* b, const char* c) { + size_t alen = strlen(a); + size_t blen = strlen(b); + size_t clen = strlen(c); + char* result = malloc(alen + blen + clen + 1); + sprintf(result, "%s%s%s", a, b, c); + return result; +} + +char* join4(const char* a, const char* b, const char* c, const char* d) { + size_t alen = strlen(a); + size_t blen = strlen(b); + size_t clen = strlen(c); + size_t dlen = strlen(d); + char* result = malloc(alen + blen + clen + dlen + 1); + sprintf(result, "%s%s%s%s", a, b, c, d); + return result; +} + +char* join5(const char* a, const char* b, const char* c, const char* d, const char* e) { + size_t alen = strlen(a); + size_t blen = strlen(b); + size_t clen = strlen(c); + size_t dlen = strlen(d); + size_t elen = strlen(e); + char* result = malloc(alen + blen + clen + dlen + elen + 1); + sprintf(result, "%s%s%s%s%s", a, b, c, d, e); + return result; +} + +char* join6(const char* a, const char* b, const char* c, const char* d, const char* e, const char* f) { + size_t alen = strlen(a); + size_t blen = strlen(b); + size_t clen = strlen(c); + size_t dlen = strlen(d); + size_t elen = strlen(e); + size_t flen = strlen(f); + char* result = malloc(alen + blen + clen + dlen + elen + flen + 1); + sprintf(result, "%s%s%s%s%s%s", a, b, c, d, e, f); + return result; +} + +const char* skip_arg(const char* cp) { + while(*cp && !isspace(*cp)) + cp++; + return cp; +} + +char* skip_spaces(const char* str) { + while(isspace(*str)) + str++; + return(char*)str; +} + +int count_argc(const char* str) { + int count = 0; + while(*str) { + str = skip_spaces(str); + if(!*str) + continue; + count++; + str = skip_arg(str); + } + return count; +} + +void argv_free(char** argv) { + for(char** p = argv; *p; p++) { + free(*p); + *p = NULL; + } + free(argv); +} + +char** argv_split(const char* str, int* argc_out) { + int argc = count_argc(str); + char** result = calloc(argc+1, sizeof(*result)); + if(result == NULL) + return result; + if(argc_out) + *argc_out = argc+1; + char** argvp = result; + while(*str) { + str = skip_spaces(str); + if(!*str) + continue; + const char* p = str; + str = skip_arg(str); + char* t = strndup(p, str-p); + if(t == NULL) { + perror("strndup"); + argv_free(result); + return NULL; + } + *argvp++ = t; + } + *argvp = NULL; + return result; +} + +int which(const char* program_name, char* out_full_program, int max_path) { + assert(out_full_program); + assert(max_path > 0); + // sanity check - maybe program_name is actually a full-path to begin with + if(access(program_name, X_OK) == 0) { + snprintf(out_full_program, max_path, "%s", program_name); + return 0; + } + char* path = getenv("PATH"); + if (path == NULL) { + log_error("PATH environment variable not found."); + return -1; + } + char* path_cpy = strdup(path); + char* dir = strtok(path_cpy, ":"); + char full_path[PATH_MAX]; + while(dir != NULL) { + snprintf(full_path, sizeof(full_path), "%s/%s", dir, program_name); + if(access(full_path, X_OK) == 0) { + snprintf(out_full_program, max_path, "%s", full_path); + free(path_cpy); + return 0; + } + dir = strtok(NULL, ":"); + } + log_error("'%s' not found in PATH", program_name); + free(path_cpy); + return -1; }