commit 01262bccff9d3409b7a5a9eac5015733fa23ae73 from: Sergey Bronnikov date: Wed Mar 24 10:00:26 2021 UTC moved runtest to a separate repository https://github.com/ligurio/runtest commit - d70548bb7a2f9afcf7e77c76412d66f9ec9d9392 commit + 01262bccff9d3409b7a5a9eac5015733fa23ae73 blob - 6e92f57d4647a41113f50e94f59047114f7e1d81 (mode 644) blob + /dev/null --- tools/runtest/.gitignore +++ /dev/null @@ -1 +0,0 @@ -tags blob - 12c2c1ab7d33da797b4a830962cf246ecbd54852 (mode 644) blob + /dev/null --- tools/runtest/Makefile +++ /dev/null @@ -1,26 +0,0 @@ -CC = cc -CFLAGS = -I . -CFLAGS += -W -Wall -Wextra -Wfloat-equal -CFLAGS += -Wundef -Wpointer-arith -Wcast-align -Wshadow -CFLAGS += -Wstrict-overflow=5 -Wwrite-strings -Waggregate-return -CFLAGS += -Wswitch-enum -Wunreachable-code -Winit-self -CFLAGS += -Wno-unused-parameter -Werror -pedantic -O3 -LDFLAGS = - -BASE_SOURCES = utils.c test_list.c -SOURCES = main.c $(BASE_SOURCES) -OBJS = $(SOURCES:.c=.o) -EXECUTABLE = runtest - -all: $(SOURCES) $(EXECUTABLE) - -$(EXECUTABLE): $(OBJS) - $(CC) $(LDFLAGS) $(OBJS) -o $@ - -.c.o: - $(CC) $(CFLAGS) -c $< -o $@ - -clean: - rm -rf $(EXECUTABLE) $(OBJS) - -.PHONY: clean blob - 77ac5e72ba8a5c883a778a4db5e9bf4e13690227 (mode 644) blob + /dev/null --- tools/runtest/README.md +++ /dev/null @@ -1,70 +0,0 @@ -RUNTEST(1) - General Commands Manual - -# NAME - -**runtest** - is a program for running OpenBSD regression tests. - -# SYNOPSIS - -**runtest** -\[**-h**] -\[**-d** *directory*] -\[**-f** *filter*] -\[**-l** *list*] -\[**-t** *timeout*] -\[**-o** *report*] -\[**-v**] - -# DESCRIPTION - -The -**runtest** -executes regression tests. -The options are as follows: - -**-d** - -> Directory where -> **runtest** -> will search tests. - -**-f** - -> Specify regular expression to filtering tests. - -**-l** - -> Lists found tests. - -**-t** - -> Timeout. - -**-o** - -> Filename where -> **runtest** -> will output TAP-compliant test results. - -# STANDARDS - -M. G. Schwern, A. Lester, A. Armstrong, -*TAP 13 - The Test Anything Protocol v13*, -2003-2007. - -# EXIT STATUS - -The **runtest** utility exits 0 on success, and >0 if an error occurs. - -# SEE ALSO - -regex(3) - -# AUTHORS - -The -**runtest** -utility was written by -Sergey Bronnikov - -Debian - December 28, 2018 blob - 3d8e644760b3a07b2b6261cfd6febe830153cb95 (mode 644) blob + /dev/null --- tools/runtest/main.c +++ /dev/null @@ -1,97 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils.h" - -#define DEFAULT_DIRECTORY "/usr/src" -#define DEFAULT_TIMEOUT 300 -#define DEFAULT_EXEC_FILE "Makefile" - -static inline void print_usage(FILE *stream, char *progname) { - fprintf(stream, - "Usage: %s [-d directory] [-f filter] [-l list] [-t timeout]" - " [-o report] [-h]\n", - progname); -} - -int main(int argc, char *argv[]) { - int opt; - int rc; - - struct test_list *tests = NULL; - struct test_options opts; - - opts.directory = strdup(DEFAULT_DIRECTORY); - opts.exec_file = strdup(DEFAULT_EXEC_FILE); - opts.filter = NULL; - opts.list = 0; - opts.report = NULL; - opts.tests = NULL; - opts.timeout = DEFAULT_TIMEOUT; - - while ((opt = getopt(argc, argv, "d:f:lt:o:h")) != -1) { - switch (opt) { - case 'd': - free(opts.directory); - opts.directory = realpath(optarg, NULL); - break; - case 'f': - opts.filter = strdup(optarg); - break; - case 'l': - opts.list = 1; - break; - case 't': - opts.timeout = atoi(optarg); - break; - case 'h': - print_usage(stdout, argv[0]); - exit(0); - break; - case 'o': - opts.report = strdup(optarg); - break; - default: - print_usage(stdout, argv[0]); - exit(1); - break; - } - } - - fprintf(stdout, "Searching tests in %s\n", opts.directory); - tests = test_discovery(opts.directory, opts.exec_file); - if (tests == NULL || test_list_length(tests) == 0) { - fprintf(stderr, MSG_TESTS_NOT_FOUND); - return 1; - } - - if (opts.filter) { - filter_tests(tests, opts.filter); - } - - if (opts.list) { - print_tests(tests, stdout); - return 0; - } - - rc = run_tests(tests, opts, argv[0], stdout, stderr); - - if (opts.report) { - FILE *fp = NULL; - if ((fp = fopen(opts.report, "w+")) == NULL) - return 1; - print_report(tests, fp); - } - - free_tests(tests); - - return rc; -} blob - 2b729a6adb79de381c98f42baeabd6c8a6c294a2 (mode 644) blob + /dev/null --- tools/runtest/runtest.1 +++ /dev/null @@ -1,71 +0,0 @@ -.\" $Id$ -.\" -.\" Copyright (c) 2020 Sergey Bronnikov -.\" -.\" Permission to use, copy, modify, and distribute this software for any -.\" purpose with or without fee is hereby granted, provided that the above -.\" copyright notice and this permission notice appear in all copies. -.\" -.\" THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -.\" WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -.\" MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -.\" ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -.\" WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -.\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -.\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -.\" -.Dd $Mdocdate: October 21 2020 $ -.Dt RUNTEST 1 -.Os -.Sh NAME -.Nm runtest -.Nd is a program for running OpenBSD regression tests. -.Sh SYNOPSIS -.Nm -.Op Fl h -.Op Fl d Ar directory -.Op Fl f Ar filter -.Op Fl l Ar list -.Op Fl t Ar timeout -.Op Fl o Ar report -.Op Fl v -.Sh DESCRIPTION -The -.Nm -executes regression tests. -The options are as follows: -.Bl -tag -width Dssmacro=value -.It Fl d -Directory where -.Nm -will search tests. -.It Fl f -Specify regular expression to filtering tests. -.It Fl l -Lists found tests. -.It Fl t -Timeout. -.It Fl o -Filename where -.Nm -will output TAP-compliant test results. -.El -.Sh STANDARDS -.Rs -.%A M. G. Schwern, A. Lester, A. Armstrong -.%D 2003-2007 -.%T TAP 13 - The Test Anything Protocol v13 -.Re -.Pp -.Sh EXIT STATUS -.Ex -std -.Sh SEE ALSO -.Xr bsd.regress.mk 5 -.Xr regex 3 -.Xr make 1 -.Sh AUTHORS -.An -nosplit -The -.Nm -utility was written by -.An Sergey Bronnikov blob - c7e1f5d91eaeb6aaa801e626df49e9f19d50d140 (mode 644) blob + /dev/null --- tools/runtest/test_list.c +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright © 2018 Sergey Bronnikov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include - -#include "test_list.h" - -void free_tests(struct test_list *tests) { - struct test *t; - while ((t = TAILQ_FIRST(tests))) { - TAILQ_REMOVE(tests, t, entries); - free_test(t); - } -} - -void free_test(struct test *t) { - if (t->name) - free((char *)t->name); - if (t->path) - free((char *)t->path); - if (t->comment) - free((char *)t->comment); - if (t->system_out) - free((char *)t->system_out); - if (t->system_err) - free((char *)t->system_err); - free(t); -} blob - f65d4c7000cafb27ae4f7ee00fc10629f65e4a23 (mode 644) blob + /dev/null --- tools/runtest/test_list.h +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright © 2018 Sergey Bronnikov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef TEST_LIST_H -#define TEST_LIST_H - -#include - -enum test_status { - STATUS_OK, /* TestAnythingProtocol */ - STATUS_NOTOK, /* TestAnythingProtocol */ - STATUS_MISSING, /* TestAnythingProtocol */ - STATUS_TODO, /* TestAnythingProtocol */ - STATUS_SKIP, /* TestAnythingProtocol */ -}; - -struct test { - enum test_status status; - const char *path; - char *name; - const char *comment; - const char *system_out; - const char *system_err; - TAILQ_ENTRY(test) entries; -}; - -TAILQ_HEAD(test_list, test); - -void free_tests(struct test_list *tests); -void free_test(struct test *t); - -#endif /* TEST_LIST_H */ blob - b59497ec34521ffe00a3a79f5aed805b4bcaa33a (mode 644) blob + /dev/null --- tools/runtest/testdir/a/b/c/d/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -all: - echo "hello" blob - b59497ec34521ffe00a3a79f5aed805b4bcaa33a (mode 644) blob + /dev/null --- tools/runtest/testdir/b/c/d/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -all: - echo "hello" blob - b59497ec34521ffe00a3a79f5aed805b4bcaa33a (mode 644) blob + /dev/null --- tools/runtest/testdir/c/d/e/f/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -all: - echo "hello" blob - b59497ec34521ffe00a3a79f5aed805b4bcaa33a (mode 644) blob + /dev/null --- tools/runtest/testdir/d/e/f/g/h/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -all: - echo "hello" blob - 5ffbc8a3e51ed7ea87a36b24e67b40852f9f91e1 (mode 644) blob + /dev/null --- tools/runtest/utils.c +++ /dev/null @@ -1,335 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "test_list.h" -#include "utils.h" - -#define TIME_BUF_SIZE 1024 -#define WAIT_CHILD_POLL_TIMEOUT_MS 200 -#define WAIT_CHILD_BUF_MAX_SIZE 1024 - -static inline char *format_time(char *strtime, size_t size) { - time_t t; - struct tm *lt; - - t = time(NULL); - lt = localtime(&t); - strftime(strtime, size, "%Y-%m-%dT%H:%M", lt); - - return strtime; -} - -/* Compare files by name. */ -int entcmp(const FTSENT **a, const FTSENT **b) -{ - return strcmp((*a)->fts_name, (*b)->fts_name); -} - -/* - * Print all files in the directory tree that match the glob pattern. - * Example: test_discovery("/usr/src/test", "Makefile"); - */ -struct test_list *test_discovery(char *dir, const char *pattern) { - struct test_list *tests; - tests = calloc(1, sizeof(struct test_list)); - if (tests == NULL) { - fprintf(stderr, "malloc failed"); - return NULL; - } - TAILQ_INIT(tests); - - FTS *tree; - FTSENT *f; - char *argv[] = { dir, NULL }; - - /* - * FTS_LOGICAL follows symbolic links, including links to other - * directories. It detects cycles, so we never have an infinite - * loop. FTS_NOSTAT is because we never use f->statp. It uses - * our entcmp() to sort files by name. - */ - tree = fts_open(argv, FTS_LOGICAL | FTS_NOSTAT, entcmp); - if (tree == NULL) - err(1, "fts_open"); - - while ((f = fts_read(tree))) { - switch (f->fts_info) { - case FTS_DNR: /* Cannot read directory */ - case FTS_ERR: /* Miscellaneous error */ - case FTS_NS: /* stat() error */ - /* Show error, then continue to next files. */ - warn("%s", f->fts_path); - continue; - case FTS_DP: - /* Ignore post-order visit to directory. */ - continue; - } - - if (fnmatch(pattern, f->fts_name, FNM_PERIOD) == 0) { - struct test *t = NULL; - t = calloc(1, sizeof(struct test)); - if (t == NULL) { - fprintf(stderr, "malloc failed"); - break; - } - t->name = dirname(strdup(f->fts_path)); - t->path = strdup(f->fts_path); - TAILQ_INSERT_TAIL(tests, t, entries); - } - - /* - * A cycle happens when a symbolic link (or perhaps a - * hard link) puts a directory inside itself. Tell user - * when this happens. - */ - if (f->fts_info == FTS_DC) { - warnx("%s: cycle in directory tree", f->fts_path); - } - } - - /* fts_read() sets errno = 0 unless it has error. */ - if (errno != 0) - err(1, "fts_read"); - - if (fts_close(tree) < 0) - err(1, "fts_close"); - - return tests; -} - -int print_tests(struct test_list *tests, FILE *fp) { - if (tests == NULL || test_list_length(tests) <= 0) { - fprintf(fp, MSG_TESTS_NOT_FOUND); - return 1; - } else { - fprintf(fp, MSG_TESTS_AVAILABLE); - struct test *t = NULL; - TAILQ_FOREACH(t, tests, entries) { - if (t->status != STATUS_SKIP) - fprintf(fp, "%s\n", t->name); - } - - return 0; - } -} - -static inline void run_child(char *run_test, int fd_stdout, int fd_stderr) { - if (chdir(dirname(strdup(run_test))) == -1) { - exit(-1); - } - - char make_bin[] = "/usr/bin/make"; - char make_opt[] = "-C"; - char *argv[] = {make_bin, make_opt, run_test, NULL}; - int rc = execv(argv[0], argv); - - dup2(fd_stdout, STDOUT_FILENO); - dup2(fd_stderr, STDERR_FILENO); - - exit(rc); -} - -static inline int wait_child(const char *test_dir, const char *run_test, - pid_t pid, int timeout, int *fds, FILE **fps) { - struct pollfd pfds[2]; - struct timespec sentinel; - clockid_t clock = CLOCK_MONOTONIC; - int r; - - int timeouted = 0; - int status; - int waitflags; - - pfds[0].fd = fds[0]; - pfds[0].events = POLLIN; - pfds[1].fd = fds[1]; - pfds[1].events = POLLIN; - - if (clock_gettime(clock, &sentinel) == -1) { - clock = CLOCK_REALTIME; - clock_gettime(clock, &sentinel); - } - - while (1) { - waitflags = WNOHANG; - - r = poll(pfds, 2, WAIT_CHILD_POLL_TIMEOUT_MS); - if (r > 0) { - char buf[WAIT_CHILD_BUF_MAX_SIZE]; - ssize_t n; - - if (pfds[0].revents != 0) { - while ((n = read(fds[0], buf, WAIT_CHILD_BUF_MAX_SIZE)) > 0) - fwrite(buf, n, 1, fps[0]); - } - - if (pfds[1].revents != 0) { - while ((n = read(fds[1], buf, WAIT_CHILD_BUF_MAX_SIZE)) > 0) - fwrite(buf, n, 1, fps[1]); - } - - clock_gettime(clock, &sentinel); - } else if (timeout >= 0) { - struct timespec time; - - clock_gettime(clock, &time); - if ((time.tv_sec - sentinel.tv_sec) > timeout) { - timeouted = 1; - kill(pid, SIGKILL); - waitflags = 0; - } - } - - if (waitpid(pid, &status, waitflags) == pid) - break; - } - - if (status) { - fprintf(fps[0], "\nERROR: Exit status is %d\n", status); - if (timeouted) - fprintf(fps[0], "TIMEOUT: %s\n", test_dir); - } - - return status; -} - -int run_tests(struct test_list *tests, const struct test_options opts, - const char *progname, FILE *fp_stdout, FILE *fp_stderr) { - int rc = 0; - char time[TIME_BUF_SIZE]; - - pid_t child; - int pipefd_stdout[2]; - int pipefd_stderr[2]; - - do { - if ((rc = pipe2(pipefd_stdout, O_NONBLOCK)) == -1) - break; - - if ((rc = pipe2(pipefd_stderr, O_NONBLOCK)) == -1) { - close(pipefd_stdout[0]); - close(pipefd_stdout[1]); - break; - } - - struct test *t = NULL; - TAILQ_FOREACH(t, tests, entries) { - if (t->status == STATUS_SKIP) { - fprintf(fp_stdout, "%s SKIPPED: %s\n", format_time(time, TIME_BUF_SIZE), - t->name); - continue; - } - char *test_dir = strdup(t->name); - if (test_dir == NULL) { - rc = -1; - break; - } - - child = fork(); - if (child == -1) { - fprintf(fp_stdout, "ERROR: Fork %s\n", strerror(errno)); - rc = -1; - break; - } else if (child == 0) { - run_child(t->name, pipefd_stdout[1], pipefd_stderr[1]); - } else { - int status; - int fds[2]; - fds[0] = pipefd_stdout[0]; - fds[1] = pipefd_stderr[0]; - FILE *fps[2]; - fps[0] = fp_stdout; - fps[1] = fp_stderr; - - fprintf(fp_stdout, "%s BEGIN: %s\n", format_time(time, TIME_BUF_SIZE), - test_dir); - - status = wait_child(test_dir, t->name, child, opts.timeout, fds, fps); - if (status) - rc += 1; - - if (WIFEXITED(status)) { - if (WEXITSTATUS(status) != 0) { - t->status = STATUS_NOTOK; - } else { - t->status = STATUS_OK; - } - } - - fprintf(fp_stdout, "%s END: %s\n", format_time(time, TIME_BUF_SIZE), - test_dir); - } - } - - close(pipefd_stdout[0]); - close(pipefd_stdout[1]); - close(pipefd_stderr[0]); - close(pipefd_stderr[1]); - } while (0); - - if (rc == -1) - fprintf(fp_stderr, "run_tests fails: %s", strerror(errno)); - - return rc; -} - -int test_list_length(struct test_list *tests) { - int num = 0; - struct test *t = NULL; - TAILQ_FOREACH(t, tests, entries) { num++; } - return num; -} - -void print_report(struct test_list *tests, FILE *fp) { - - int n = 1; - fprintf(fp, "TAP version 13\n"); - fprintf(fp, "1..%d\n", test_list_length(tests)); - struct test *t = NULL; - TAILQ_FOREACH(t, tests, entries) { - dirname(t->name); - if (t->status == STATUS_OK) - fprintf(fp, "ok %d - %s\n", n, t->name); - else if (t->status == STATUS_NOTOK) - fprintf(fp, "not ok %d - %s\n", n, t->name); - else if (t->status == STATUS_SKIP) - fprintf(fp, "ok %d - %s # SKIP\n", n, t->name); - n++; - } -} - -void filter_tests(struct test_list *tests, const char *filter) { - regex_t reg; - regmatch_t match[1]; - - int r; - r = regcomp(®, filter, REG_ICASE | REG_EXTENDED); - if (r != 0) { - fprintf(stderr, "%s\n", strerror(errno)); - } - struct test *t = NULL; - TAILQ_FOREACH(t, tests, entries) { - r = regexec(®, t->name, 1, match, 0); - if (r != 0) { - t->status = STATUS_SKIP; - } - } - regfree(®); -} blob - 1a605247ceecd97ead5f17d8db2cb789af17a88c (mode 644) blob + /dev/null --- tools/runtest/utils.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef _UTILS_H_ -#define _UTILS_H_ - -#include "test_list.h" - -#define MSG_TESTS_NOT_FOUND "No tests found.\n" -#define MSG_TESTS_AVAILABLE "Found tests:\n" - -struct test_options { - char *directory; - char *filter; - int list; - int timeout; - char **tests; - char *report; - char *exec_file; -}; - -extern struct test_list *test_discovery(char *, const char *); -extern int print_tests(struct test_list *, FILE *); -extern int run_tests(struct test_list *, const struct test_options, - const char *, FILE *, FILE *); -extern int test_list_length(struct test_list *tests); - -extern void print_report(struct test_list *, FILE *); -void filter_tests(struct test_list *, const char *); - -#endif /* _UTILS_H_ */