diff --git a/.gitignore b/.gitignore index 96426ec..f115bc8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ out/ examples/ compile_commands.json .cache/ +*.tar.gz +*.log +*.zst diff --git a/Makefile b/Makefile index a21b7dc..7684936 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,13 @@ # See LICENSE file for copyright and license details. # Note: If you're confused by the makefile, I do emplore you to read the info-page: $ info make .POSIX: + NAME=sci DESCRIPTION=$(NAME) is a simple contiuous integration system. -VERSION = 0.1.0 +VERSION = 1.0.0 +PREFIX = /usr/local +MANPREFIX = $(PREFIX)/share/man + CC = gcc OUTDIR := out/ OBJDIR := out/obj @@ -14,11 +18,11 @@ CFLAGS += -DSCI_NAME="\"$(NAME)\"" CFLAGS += -DSCI_DESCRIPTION="\"$(DESCRIPTION)\"" CFLAGS += -D_POSIX_C_SOURCE=2 CFLAGS += -D_GNU_SOURCE -CFLAGS += -Wall -Werror -std=c23 -g +CFLAGS += -Wall -Werror -std=c11 -g CFLAGS += -Iinclude CFLAGS += -lpthread -luuid -.PHONY: all clean +.PHONY: all clean dist install all: out/bin/sci @@ -38,6 +42,7 @@ out/bin/sci: $(OBJ) | $(BINDIR) clean: rm -rf $(OUTDIR) + rm -rf $(NAME)-$(VERSION) $(OUTDIR): mkdir -p $@ @@ -48,20 +53,34 @@ $(OBJDIR): $(OUTDIR) $(BINDIR): $(OUTDIR) mkdir -p $@ -# dist: clean -# mkdir -p st-$(VERSION) -# cp -R FAQ LEGACY TODO LICENSE Makefile README config.mk\ -# config.def.h st.info st.1 arg.h st.h win.h $(SRC)\ -# st-$(VERSION) -# tar -cf - st-$(VERSION) | gzip > st-$(VERSION).tar.gz -# rm -rf st-$(VERSION) -# -# install: st -# mkdir -p $(DESTDIR)$(PREFIX)/bin -# cp -f st $(DESTDIR)$(PREFIX)/bin -# chmod 755 $(DESTDIR)$(PREFIX)/bin/st -# mkdir -p $(DESTDIR)$(MANPREFIX)/man1 -# sed "s/VERSION/$(VERSION)/g" < st.1 > $(DESTDIR)$(MANPREFIX)/man1/st.1 -# chmod 644 $(DESTDIR)$(MANPREFIX)/man1/st.1 -# tic -sx st.info -# @echo Please see the README file regarding the terminfo entry of st. +dist: + mkdir -p $(NAME)-$(VERSION) + cp -R \ + TODO.md README.md\ + Makefile src include\ + $(NAME)-$(VERSION) + tar -cf - $(NAME)-$(VERSION) | gzip > $(NAME)-$(VERSION).tar.gz + rm -rf $(NAME)-$(VERSION) + +# NOTE: DESTDIR is meant for making packaging easier. +# If you want to install in a different directory than the default, please +# use: # make install PREFIX=/custom/path +install: out/bin/sci + mkdir -p $(DESTDIR)$(PREFIX)/bin + # install binaries + cp -f out/bin/sci $(DESTDIR)$(PREFIX)/bin + chmod 755 $(DESTDIR)$(PREFIX)/bin/sci + # install libraries + # install services (only if system is using systemd though) + # install manpages + mkdir -p $(DESTDIR)$(MANPREFIX)/man1 + sed "s/VERSION/$(VERSION)/g" < src/sci.1 > $(DESTDIR)$(MANPREFIX)/man1/sci.1 + chmod 644 $(DESTDIR)$(MANPREFIX)/man1/sci.1 + +uninstall: + # uninstall binaries + rm -f $(DESTDIR)$(PREFIX)/bin/sci + # uninstall libraries + # uninstall services (only if system is using systemd though) + # uninstall manpages + rm -f $(DESTDIR)$(MANPREFIX)/man1/sci.1 diff --git a/PKGBUILD b/PKGBUILD new file mode 100644 index 0000000..92a5ad8 --- /dev/null +++ b/PKGBUILD @@ -0,0 +1,36 @@ +# Maintainer: Asger Gitz-Johansen +pkgname=sci +pkgver=1.0.0 +pkgrel=1 +epoch= +pkgdesc="A simple / minimal CI (Continuous Integration) system" +arch=('x86_64') # TODO: also arm64 when you're not tired +url="gitea.local:3000/agj/sci" +license=('GPL-3.0-or-later') # TODO: add LICENSE file and header +groups=() +depends=("glibc" + "util-linux-libs") +makedepends=() +checkdepends=() +optdepends=() +provides=() +conflicts=() +replaces=() +backup=() +options=() +install= +changelog= +source=("$pkgname-$pkgver.tar.gz") +noextract=() +sha256sums=("948092bdcc3591afcdc205263832a06c838aa9d524c762b061e91cffa04b7d63") +validpgpkeys=() + +build() { + cd "$pkgname-$pkgver" + make +} + +package() { + cd "$pkgname-$pkgver" + make DESTDIR="$pkgdir/" PREFIX="/usr" install +} diff --git a/README.md b/README.md index 8ea179a..f98fd8b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,68 @@ -# Suckless/Simple Continous Integration +# Simple CI (Continous Integration) Jenkins, Travis, GitHub Actions, GitLab CI. The list goes on. -This is a minimal tool for fulfilling the CI (Continous Integration) use case. +There are many CI systems. +All of them overcomplicate an extremely simple use case: +"I want to trigger custom scripts (usually just a list of simple shell commands) on a server. +Either automatically, or manually. +And then I want to be follow and see any errors that may happen." + +There's no need to invent the wheel 10 different times. +Most of the actions required by the CI use case can be handled with most UNIX tools such as `tail`, +`cat`, `inotify` etc. +At the core of CI must be a triggering system that can be dynamically configured to run a pipeline script. + +This program provides that simple triggering system where you simply `touch` a file, and the associated +pipeline will then be executed. +You can then follow the log via `tail`. + +The operation of sci is configured through a pipelines.conf configuration file (see sci(7) for +configuration language details) and each pipeline will have an associated pipeline trigger file +that can be By default, pipeline triggers are placed in /tmp/sci but this can be overridden with +the `-T` flag (WIP feature). For more commandline options see the manpage `sci(1)` or use the `--help` option. + +## Dependencies +Just GNU/Linux! `sci` is just using the POSIX api, so it should work with *NIX, but it has only been tested on GNU/Linux +systems. +NOTE: Not tested with `muslc` systems yet, `sci` use some libc GNU extensions, but it would be nice to support alpine +linux some time in the future. + +## Build +Build the project using `make`: +```sh +make +``` + +## Install +Install the project: +```sh +make +sudo make install +``` + +By default, `sci` is installed to `/usr/local` but this can be overridden by providing the `PREFIX` var: +```sh +make +sudo make install PREFIX=/some/path +``` + +## Package +`sci` can be packaged for multiple distributions: + +### Arch Linux +It is recommended that you use an arch linux distribution when building. + +```sh +``` + +### Debian +It is recommended that you use the `deb-builder.dockerfile` docker image to build the debian image. +```sh +``` + +#### Building the debian builer docker image +```sh +docker build -t debbuilder deb-builder.dockerfile . +``` ## Brainstorm If you dont want to congest your CI server. Too bad. Write faster ci suites. (TODO: implement runners) @@ -75,37 +137,3 @@ I choose `c`! I also choose `Makefile`s! - Just to force myself to use another build system than CMake! If you want `compile_commands.json` files, you should use [bear](https://github.com/rizsotto/Bear) as it works well - -### Progress - - [x] Zeroth things first, let's create a simple CLI application with `--verbosity VAL` and `--help` options. - - [x] First things first, let's implement something that reacts when some provided file changes (not poll please). - - [x] Second things second, implement a simple logging system with differing levels of verbosity and configurable - output file using cli options. - - [x] Third things third, implement a thing that simultaneously watches two different files (multithreading). - it should be cancellable with ctrl+c, but it should just contiuously print event notifications. - - [x] Fourth things fourth, implement a prototype that reads a space-separated file and populates a struct. - - [x] Fifth things fifth, implement a prototype that spawns a new thread that executes a shell command. - - [ ] Sixth things sixth, daemonize it! - - [ ] Seventh things seventh, package the sucker (arch, debian, alpine, docker) - - [ ] Eight things eight, try it out! - maybe even write the python webhook extension. - - [ ] Ninth things ninth, fix bugs, see below - - [ ] Tenth things tenth, write manpages - - [ ] Eleventh things last, release! - -#### Bugs - - [ ] command output is being inherited. It should be piped into some random log-file - - [ ] pretty sure that `ctrl+c` / SIGINT is not graceful yet. - - [ ] missing license - - [ ] I am deliberately not using `Restart=on-failure` in the `scid.service` file because we are using `Type=exec` - and not `Type=notify` (yet) - which would require a `sd_notify` call of `READY=1` (see `man systemd.service`) - -### Note Regarding `inotify` usage -From the manpage: -``` -With careful programming, an application can use inotify to efficiently monitor and cache the state of a set of -filesystem objects. However, robust applications should allow for the fact that bugs in the monitoring logic or races -of the kind described below may leave the cache inconsistent with the filesystem state. It is probably wise to do some -consistency checking, and re‐build the cache when inconsistencies are detected. -``` -i.e., we should _also_ poll the watched files every once in a while (maybe once per minute? idk) to ensure that we catch -all events. diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..780e494 --- /dev/null +++ b/TODO.md @@ -0,0 +1,61 @@ +### Progress + - [x] Zeroth things first, let's create a simple CLI application with `--verbosity VAL` and `--help` options. + - [x] First things first, let's implement something that reacts when some provided file changes (not poll please). + - [x] Second things second, implement a simple logging system with differing levels of verbosity and configurable + output file using cli options. + - [x] Third things third, implement a thing that simultaneously watches two different files (multithreading). + it should be cancellable with ctrl+c, but it should just contiuously print event notifications. + - [x] Fourth things fourth, implement a prototype that reads a space-separated file and populates a struct. + - [x] Fifth things fifth, implement a prototype that spawns a new thread that executes a shell command. + - [x] Sixth things sixth, daemonize it! + - [ ] Seventh things seventh, package the sucker (arch, debian, alpine, docker) + - [x] archlinux + - https://wiki.archlinux.org/title/Creating_packages + - [ ] debian + - see `/home/agj/documents/Projects/packaging/deb-packaging-tutorial.pdf` + - just use docker. + - [ ] alpine + - [ ] docker + - [ ] Eight things eight, try it out! - maybe even write the python webhook extension. + - [ ] Ninth things ninth, fix bugs, see below + - [ ] Tenth things tenth, write manpages, choose license + - [ ] Eleventh things Eleventh, polish + - [ ] Twelveth things last, release! + - Setup gitea.gtz.dk (will learn you how to set up subdomains (useful for shop.gtz.dk)) + +BOOKMARK: You were reading :Man system.unit and :Man systemd.service as preperation on making a systemd unit file +This will be needed for the .deb package, as well as the arch linux package. +alpine linux is using OpenRC (cool), which complicates things a little bit, but shouldn't be too bad. The wiki is +generally really well written. Otherwise, I am sure that both wiki.gentoo and wiki.archlinux have great pages too +docker is super easy, just make a dockerfile - only concern is the trigger files. + +#### Bugs / Missing Features + - [x] command output is being inherited. It should be piped into some random log-file + - [ ] pretty sure that `ctrl+c` / SIGINT is not graceful yet. + - [ ] missing license (heavily considering GPLv3) + - [ ] pipeline scripts should be executed in a unique `/tmp` dir + - [ ] Some way for third parties to see which pipelines are currently running and their status. + - Could be as simple as looking in the logs directory. + - How to mark a run as failed / success / warn? + - Third parties may need to extract artifacts. + or maybe the scripts themselves would upload artifacts? + - [ ] I am deliberately not using `Restart=on-failure` in the `scid.service` file because we are using `Type=exec` + and not `Type=notify` (yet) - which would require a `sd_notify` call of `READY=1` (see `man systemd.service`) + +### Note Regarding `inotify` usage +From the manpage: +``` +With careful programming, an application can use inotify to efficiently monitor and cache the state of a set of +filesystem objects. However, robust applications should allow for the fact that bugs in the monitoring logic or races +of the kind described below may leave the cache inconsistent with the filesystem state. It is probably wise to do some +consistency checking, and re‐build the cache when inconsistencies are detected. +``` +i.e., we should _also_ poll the watched files every once in a while (maybe once per minute? idk) to ensure that we catch +all events. + +### Useful manpages + - `hier(7)` + - `daemon(7)` and `daemon(3)` + - `PKGBUILD(5)` + - `info make` + - `https://wiki.archlinux.org/title/Creating_packages` diff --git a/deb-builder.dockerfile b/deb-builder.dockerfile new file mode 100644 index 0000000..dcd7c0e --- /dev/null +++ b/deb-builder.dockerfile @@ -0,0 +1,18 @@ +FROM debian:latest +# TODO: Remove busybox from this list once you're done experimenting (you only need vi) +RUN apt-get update ; apt-get install -y build-essential devscripts dh-make busybox +ARG DOCKER_USER=deb + +# Set user and group +ARG user=appuser +ARG group=appuser +ARG uid=1000 +ARG gid=1000 +RUN groupadd -g ${gid} ${group} +RUN useradd -u ${uid} -g ${group} -s /bin/sh -m ${user} # <--- the '-m' create a user home directory +ENV USER=${user} + +# Switch to user +USER ${uid}:${gid} + +ENTRYPOINT ["bash", "--login"] diff --git a/include/log.h b/include/log.h index 0b0d3d3..510325d 100644 --- a/include/log.h +++ b/include/log.h @@ -1,6 +1,7 @@ #ifndef SCI_LOG_H #define SCI_LOG_H #include +#include enum { LOG_TRACE = 4, diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100644 index 0000000..7d29d9c --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -e # exit immediately on error + +git clone -b dev "$SCI_PIPELINE_URL" "$SCI_PIPELINE_NAME" +echo "clone success" + +cd "$SCI_PIPELINE_NAME" + +cmake -B build +echo "configure success" + +cmake --build build +echo "build success" + +cpack build +echo "packaging success" + +# TODO: upload artifacts to some artifact store +# curl "build/dist/your-package.deb" > ftp://example.com/artifacts +# echo "artifacts upload success" diff --git a/src/log.c b/src/log.c index 8df59ef..a01dfb8 100644 --- a/src/log.c +++ b/src/log.c @@ -72,7 +72,7 @@ void log_log(const char* file, int line, int level, const char* fmt, ...) { else fprintf(g_log_settings.out_file, "%s %-5s %s:%d: ", timestamp, level_name, file, line); va_list args; - va_start(args, format); + va_start(args, fmt); vfprintf(g_log_settings.out_file, fmt, args); va_end(args); fprintf(g_log_settings.out_file, "\n"); diff --git a/src/main.c b/src/main.c index 1b48403..8e03655 100644 --- a/src/main.c +++ b/src/main.c @@ -148,7 +148,7 @@ int main(int argc, char** argv) { } if(!args.config_file.has_value) { - fprintf(stderr, "no file provided see -h for usage\n"); + fprintf(stderr, "no pipeline config file provided see -h for usage\n"); exit(EXIT_FAILURE); } @@ -170,11 +170,6 @@ int main(int argc, char** argv) { pool = threadpool_create(args.executors); per_line(args.config_file.value, &config_interpret_line); - // BOOKMARK: You were reading :Man system.unit and :Man systemd.service as preperation on making a systemd unit file - // This will be needed for the .deb package, as well as the arch linux package. - // alpine linux is using OpenRC (cool), which complicates things a little bit, but shouldn't be too bad. The wiki is - // generally really well written. Otherwise, I am sure that both wiki.gentoo and wiki.archlinux have great pages too - // docker is super easy, just make a dockerfile - only concern is the trigger files. pipeline_loop(); threadpool_destroy(pool); } diff --git a/src/sci.1 b/src/sci.1 new file mode 100644 index 0000000..ab6342c --- /dev/null +++ b/src/sci.1 @@ -0,0 +1,57 @@ +.\" Hey, EMACS: -*- nroff -*- +.\" (C) Copyright 2024 unknown , +.\" +.\" First parameter, NAME, should be all caps +.\" Second parameter, SECTION, should be 1-8, maybe w/ subsection +.\" other parameters are allowed: see man(7), man(1) +.TH Sci SECTION "August 18 2024" +.\" Please adjust this date whenever revising the manpage. +.\" +.\" Some roff macros, for reference: +.\" .nh disable hyphenation +.\" .hy enable hyphenation +.\" .ad l left justify +.\" .ad b justify to both left and right margins +.\" .nf disable filling +.\" .fi enable filling +.\" .br insert line break +.\" .sp insert n+1 empty lines +.\" for manpage-specific macros, see man(7) +.SH NAME +sci \- program to do something +.SH SYNOPSIS +.B sci +.RI [ options ] " files" ... +.br +.B bar +.RI [ options ] " files" ... +.SH DESCRIPTION +This manual page documents briefly the +.B sci +and +.B bar +commands. +.PP +.\" TeX users may be more comfortable with the \fB\fP and +.\" \fI\fP escape sequences to invode bold face and italics, +.\" respectively. +\fBsci\fP is a program that... +.SH OPTIONS +These programs follow the usual GNU command line syntax, with long +options starting with two dashes ('\-'). +A summary of options is included below. +For a complete description, see the Info files. +.TP +.B \-h, \-\-help +Show summary of options. +.TP +.B \-v, \-\-version +Show version of program. +.SH SEE ALSO +.BR bar (1), +.BR baz (1). +.br +The programs are documented fully by +.IR "The Rise and Fall of a Fooish Bar" , +available via the Info system. +