feat: add better shell-like command support

You can now execute any kind of program in the PATH.
You do, however, need to specify "./" if you want to execute a local
file, but that shouldn't be too big a problem.

.dockerfile

@@ -1,6 +0,0 @@
-# TODO: use alpine when available
-FROM debian:12-slim
-ADD artifacts.tar.gz /install
-RUN dpkg -i /install/artifacts/sci_*-1_amd64.deb
-RUN rm -rf /install
-ENTRYPOINT ["sci"]

.sci.sh

@@ -2,6 +2,7 @@
 set -e
 echo ">>> checking if required environment is set..."
 test -n "$DOCKER_TOKEN"
+which make
 
 echo ">>> compiling..."
 make
@@ -12,7 +13,7 @@ make dist
 SRC_SHA256=$(sha256sum "sci-${VERSION}.tar.gz" | awk '{ print $1 }')
 sed "s/SRC_SHA256/${SRC_SHA256}/g" < PKGBUILD.in > PKGBUILD
 
-# arch
+# # arch
 echo ">>> building archbuilder image..."
 docker build -t archbuilder -f arch-builder.dockerfile .
 
@@ -29,7 +30,7 @@ echo ">>> building debbuilder image..."
 docker build -t debbuilder -f deb-builder.dockerfile .
 
 echo ">>> building .deb in debbuilder docker image..."
-docker run --rm -it -v .:/src -e VERSION debbuilder sh -c '\
+docker run --rm -it -v .:/src -e VERSION -e DOCKER_TOKEN debbuilder sh -c '\
     cd && \
     mkdir -p artifacts && \
     cp /src/sci-$VERSION.tar.gz . && \
@@ -46,14 +47,8 @@ docker run --rm -it -v .:/src -e VERSION debbuilder sh -c '\
     cp ../*.tar.xz ~/artifacts && \
     cp ../*.tar.gz ~/artifacts && \
     cd && \
-    tar czf /src/artifacts.tar.gz artifacts
+    curl --user agj:$DOCKER_TOKEN \
+         --upload-file sci_$VERSION-1_amd64.deb \
+         "https://git.gtz.dk/api/packages/agj/debian/pool/bionic/main/upload"
 '
-
+# TODO: push-user should be some sci-bot or something, not your account. This will do for now though
-echo ">>> building sci docker image..."
-export OWNER="git.gtz.dk/agj"
-docker build -t ${OWNER}/sci:${VERSION} -t ${OWNER}/sci:latest -f .dockerfile .
-
-echo ">>> pushing latest docker image..."
-# TODO: user should be some sci-bot or something, not your account. This will do for now though
-docker login git.gtz.dk -u agj -p "$DOCKER_TOKEN"
-docker push ${OWNER}/sci:latest

Makefile

@@ -37,6 +37,8 @@ OBJ += out/obj/util.o
 OBJ += out/obj/pipeline.o
 OBJ += out/obj/threadlist.o
 OBJ += out/obj/threadpool.o
+OBJ += out/obj/argv_split.o
+OBJ += out/obj/which.o
 out/bin/sci: $(OBJ) | $(BINDIR)
 	$(CC) -o $@ $^ $(CFLAGS)
 

TODO.md

@@ -17,15 +17,17 @@
     - [-] ~~alpine~~ later.
     - [-] ~~docker~~ later.
  - [ ] Eight things eight, try it out! - maybe even write the python webhook extension.
+  - [ ] Port this document to gitea issue tracking
  - [ ] Ninth things ninth, fix bugs, see below
  - [ ] Tenth things tenth, write manpages, choose license
  - [ ] Eleventh things Eleventh, polish
  - [ ] Twelveth things last, release!
-   - Setup gitea.gtz.dk (will learn you how to set up subdomains (useful for shop.gtz.dk))
+   - [x] Setup git.gtz.dk (will learn you how to set up subdomains (useful for shop.gtz.dk))
+ - [ ] -1th things -1th, write a blog post about the tool (also set up your blog.gtz.dk)
 
-BOOKMARK: okay. Now it feels like it's getting complicated. I want to run `sci` in a docker container. But that means
+Okay. Now it feels like it's getting complicated. I want to run `sci` in a docker container. But that means
 that the build-threads also run in that docker container - meaning the container should have all the build dependencies
-installed and we all know where that rabbithole goes. 9-30YB docker images with about a trillion unique build systems.
+installed and we all know where that rabbithole goes. 9-30YiB docker images with about a trillion unique build systems.
 Let's not do that.
 The only alternative I can see is that the `sci` service is just not dockerized. The pipeline scripts can easily be
 dockerized themselves. Just have a `scripts/wget-src-dist-and-sci-sh-dockerized.sh` with `arg1` being the docker image
@@ -33,12 +35,24 @@ to use?
 ```sh
 #!/bin/sh
 wget "$SCI_PIPELINE_URL"
-docker run --rm -it --mount type=bind,source="$(pwd)"/thefileyouwgot.tar.gz,target=/thefileyouwgot.tar.gz,readonly --entrypoint sh $2
+docker run --rm -it -v .:/src -w /src $@
 ```
 Or something like that... Perhaps we can figure something out with an inline `ADD`, that also extracts the archive in
 the container or something. This approach is cleaner IMO. You can also more easily edit the `pipelines.conf` file if you
 need to.
 
+The aforementioned rabbithole went like this:
+ - Let's say that `sci` is run inside a docker container.
+   This would make it very easy to deploy, but:
+ - Since pipelines are executed in the same environment as `sci`, either:
+   The `sci` container must be all-encompassing. i.e. it contains every single build system and scriptling language that
+   could possibly be used by any kind of user or; all pipelines must be run from a docker container themselves, meaning
+   that the `sci` container must have `dind`-privileges. Either option is suboptimal and will lock users into one way of
+   using `sci`, which is bad.
+ - Conclusion: Fuck docker. All environment management is delegated to the user and is not `sci`'s responsibility!
+   `sci` will always be run on the ci-machine itself, unless a user has provided a custom docker image, which is fine
+   and doesn't burden the `sci` project.
+
 You were getting the following `pipelines.conf` file to work:
 ```
 scih-dev ssh://git@git.gtz.dk:222/agj/scih.git scih-onpush /etc/sci/scripts/git-clone-and-run-sci-sh.sh
@@ -69,7 +83,7 @@ docker is super easy, just make a dockerfile - only concern is the trigger files
  - [ ] Custom environment variables passed to the pipelines on invokation should be possible.
  - [ ] Listener threads should be killed and restarted (worker pool should just chug along) when pipeline config file
        has changed during runtime. Should be disableable with `--no-hot-reload-config` - i.e. on by default.
- - [ ] `docker stop` is very slow. I am probably not handling signals properly yet.
+ - [x] ~~`docker stop` is very slow. I am probably not handling signals properly yet.~~ native docker is abandonned
  - [x] It seems that `-v 4` is segfaulting when running release builds, maybe the logger just cant find the source file?
        Nope. I just wrote some bad code (inverted NULL check).
 

include/argv_split.h

@@ -0,0 +1,24 @@
+/**
+ * sci - a simple ci system
+   Copyright (C) 2024 Asger Gitz-Johansen
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+#ifndef SCI_ARGV_SPLIT_H
+#define SCI_ARGV_SPLIT_H
+
+char** create_argv_shell(const char* str, int* argc_out);
+void argv_free(char** argv);
+
+#endif

include/which.h

@@ -0,0 +1,23 @@
+/**
+ * sci - a simple ci system
+   Copyright (C) 2024 Asger Gitz-Johansen
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+#ifndef SCI_WHICH_H
+#define SCI_WHICH_H
+
+int which(const char* program_name, char* out_full_program, int max_path);
+
+#endif

scripts/README.md

@@ -0,0 +1,3 @@
+# sci default scripts
+This directory contains some default scripts that may or may not be useful to you.
+Most of the scripts are fairly simple, but should be installed as part of the sci installation process.

scripts/git-clone-and-run-sci-sh.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-set -e
-echo ">>> cloning..."
-git clone $SCI_PIPELINE_URL $SCI_PIPELINE_NAME
-cd $SCI_PIPELINE_NAME
-
-echo ">>> running .sci.sh..."
-time sh .sci.sh

scripts/git-clone-and-sci.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -ex  # print all that we're doing (no need for echo's)
+tmpdir=$(mktemp -d)
+git clone --depth=1 --recurse-submodules --shallow-submodules -b $1 "$SCI_PIPELINE_URL" "$tmpdir"
+shift
+cd "$tmpdir"
+sh .sci.sh
+cd -

scripts/wget-and-sci.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+# NOTE: This script assumes that the url is a .tar.gz file.
+# TODO: check if $# is >= 1 and give a warning that the extract dir should be provided.
+set -ex  # print all that we're doing (no need for echo's)
+tmpdir=$(mktemp -d)
+wget "$SCI_PIPELINE_URL" -P "$tmpdir"
+cd "$tmpdir"
+tar xf *.tar.gz
+cd $1
+sh .sci.sh
+cd -
+rm -rf "$tmpdir"

src/argv_split.c

@@ -0,0 +1,81 @@
+/**
+ * sci - a simple ci system
+   Copyright (C) 2024 Asger Gitz-Johansen
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+#include "argv_split.h"
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static const char* skip_arg(const char* cp) {
+	while(*cp && !isspace(*cp))
+		cp++;
+	return cp;
+}
+
+static char* skip_spaces(const char* str) {
+	while(isspace(*str))
+		str++;
+	return(char*)str;
+}
+
+static int count_argc(const char* str) {
+	int count = 0;
+	while(*str) {
+		str = skip_spaces(str);
+		if(!*str)
+			continue;
+		count++;
+		str = skip_arg(str);
+	}
+	return count;
+}
+
+void argv_free(char** argv) {
+	for(char** p = argv; *p; p++) {
+		free(*p);
+		*p = NULL;
+	}
+	free(argv);
+}
+
+char** create_argv_shell(const char* str, int* argc_out) {
+	int argc = count_argc(str);
+	char** result = calloc(argc+1, sizeof(*result));
+	if(result == NULL)
+		return result;
+	if(argc_out)
+		*argc_out = argc+1;
+	char** argvp = result;
+	while(*str) {
+		str = skip_spaces(str);
+		if(!*str)
+			continue;
+		const char* p = str;
+		str = skip_arg(str);
+		char* t = strndup(p, str-p);
+		if(t == NULL) {
+			perror("strndup");
+			argv_free(result);
+			return NULL;
+		}
+		*argvp++ = t;
+	}
+	*argvp = NULL;
+	return result;
+}
+// sci-release https://git.gtz.dk/agj/sci/archive/main.tar.gz manual "scripts/wget-and-sci.sh sci"

src/main.c

@@ -15,16 +15,20 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
+#include "argv_split.h"
 #include "cli.h"
 #include "log.h"
 #include "notify.h"
 #include "pipeline.h"
 #include "threadpool.h"
 #include "util.h"
+#include "which.h"
 #include <fcntl.h>
+#include <linux/limits.h>
 #include <spawn.h>
 #include <stdlib.h>
 #include <sys/stat.h>
+#include <unistd.h>
 #include <uuid/uuid.h>
 #include <wait.h>
 
@@ -62,8 +66,10 @@ void executor(void* data) {
 
     // Create logfile path
     optional_int fd = open_logfile(pipeline_id);
-    if(!fd.has_value)
+    if(!fd.has_value) {
+        log_error("could not open log file - not starting pipeline");
         return;
+    }
 
     // spawn the process
     pid_t pid;
@@ -72,24 +78,35 @@ void executor(void* data) {
     posix_spawn_file_actions_adddup2(&actions, fd.value, STDOUT_FILENO);
     posix_spawn_file_actions_adddup2(&actions, fd.value, STDERR_FILENO);
     const pipeline_event* const e = data;
+    char* path = join("PATH=", getenv("PATH"));
     char* name = join("SCI_PIPELINE_NAME=", e->name);
     char* url = join("SCI_PIPELINE_URL=", e->url);
     char* trigger = join("SCI_PIPELINE_TRIGGER=", e->trigger);
     char* id = join("SCI_PIPELINE_ID=", pipeline_id);
-    char* envp[] = { name, url, trigger, id, NULL };
+    char* envp[] = { path, name, url, trigger, id, NULL };
-    char* argv[] = { "/bin/sh", "-c", e->command, NULL };
+    int argc;
-    if(posix_spawn(&pid, "/bin/sh", &actions, NULL, argv, envp) != 0) {
+    char** argv = create_argv_shell(e->command, &argc);
+    log_trace("executing pipeline %s with argv:", e->name);
+    for(int i = 0; i < argc; i++)
+        log_trace("  \"%s\"", argv[i]);
+    char arg0[PATH_MAX];
+    if(which(argv[0], arg0, PATH_MAX) == -1)
+        goto end;
+    if(posix_spawn(&pid, arg0, &actions, NULL, argv, envp) != 0) {
         perror("posix_spawn");
-        goto end; // I know. The raptors have picked up the scent. I'll just have to mask it with more poopy code.
+        goto end; // I know. The raptors have picked up the scent. I'll just have to mask it with more stinky code.
     }
-    log_trace("{%s} (%s) spawned", pipeline_id, e->name);
+    log_info("{%s} (%s) spawned", pipeline_id, e->name);
 
     // Wait for process to complete
     int status;
     waitpid(pid, &status, 0);
-    if(WIFEXITED(status))
+    log_info("{%s} (%s) exited with status %d", pipeline_id, e->name, status);
-        log_trace("{%s} (%s) exited with status %d", pipeline_id, e->name, WEXITSTATUS(status));
+    char buf[32];
+    sprintf(buf, "exited with status %d", status);
+    write(fd.value, buf, strnlen(buf, 32));
 end:
+    argv_free(argv);
     close(fd.value);
     free(pipeline_id);
     free(name);

src/pipeline.c

@@ -40,7 +40,11 @@ optional_pipeline_conf pipeline_create(const char* config_line) {
             break;
         off = pmatch[0].rm_so + (cursor - config_line);
         len = pmatch[0].rm_eo - pmatch[0].rm_so;
-        opts[i] = strndup(config_line + off, len);
+        // Cut off the "-s if it is string-enclosed
+        if(config_line[off] == '"' && config_line[off+len-1] == '"')
+            opts[i] = strndup(config_line + off+1, len-2);
+        else
+            opts[i] = strndup(config_line + off, len);
         cursor += pmatch[0].rm_eo;
     }
     if(i != 4) {

src/which.c

@@ -0,0 +1,54 @@
+/**
+ * sci - a simple ci system
+   Copyright (C) 2024 Asger Gitz-Johansen
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+#include "log.h"
+#include <assert.h>
+#include <linux/limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+int which(const char* program_name, char* out_full_program, int max_path) {
+    assert(out_full_program);
+    assert(max_path > 0);
+    // sanity check - maybe program_name is actually a full-path to begin with
+    if(access(program_name, X_OK) == 0) {
+        snprintf(out_full_program, max_path, "%s", program_name);
+        return 0;
+    }
+    char* path = getenv("PATH");
+    if (path == NULL) {
+        log_error("PATH environment variable not found.");
+        return -1;
+    }
+    char* path_cpy = strdup(path);
+    char* dir = strtok(path_cpy, ":");
+    char full_path[PATH_MAX];
+    while(dir != NULL) {
+        snprintf(full_path, sizeof(full_path), "%s/%s", dir, program_name);
+        if(access(full_path, X_OK) == 0) {
+            snprintf(out_full_program, max_path, "%s", full_path);
+            free(path_cpy);
+            return 0;
+        }
+        dir = strtok(NULL, ":");
+    }
+    log_error("'%s' not found in PATH", program_name);
+    free(path_cpy);
+    return -1;
+}