nih-at/nihtest

A testing tool for command line utilities.

nihtest's Issues

Can get a build error since pymodules not explicitly listed

https://stackoverflow.com/questions/72294299/multiple-top-level-packages-discovered-in-a-flat-layout

man pages cannot be build from PyPI source

I'm busy packaging nihtest for Fedora. When trying to build the man pages, I discovered that the Makefile is missing from the PyPI tarball:

pushd manpages
make
~/build/BUILD/nihtest-1.1.0/manpages ~/build/BUILD/nihtest-1.1.0
make: *** No targets specified and no makefile found.  Stop.

Could that Makefile be shipped with the PyPI tarball?

I can use the GitHub sources for now, but I'd prefer using the PyPI sources.

Running Tests with Local install of nihtest

Am trying to install nihtest in a local folder without using a virtual environment to replicate a packaging build.
Is there some way to modify the configuration to enable the tests to run?

Modified CMakeLists.txt

cmake_minimum_required(VERSION 3.12)

# Also update version in nihtest/__main__.py and pyproject.toml
project(nihtest
        VERSION 1.1.1
        DESCRIPTION "NiH testing framework"
        HOMEPAGE_URL "https://github.com/nih-at/nihtest"
        LANGUAGES C)

enable_testing()

find_package(Python3 REQUIRED COMPONENTS Interpreter)


set(NIHTEST /home/user458/python-nihtest/nihtest-1.1.1/cbuild/test_dir/bin/nihtest)

file(GLOB SOURCES ${CMAKE_SOURCE_DIR}/nihtest/*.py)

add_custom_command(
        OUTPUT ${NIHTEST}
        COMMAND python -m pip install -q --disable-pip-version-check ${CMAKE_SOURCE_DIR} -t test_dir
        DEPENDS ${SOURCES}
)

add_custom_target(mynihtest ALL DEPENDS ${NIHTEST})

# TODO: target for wheel
add_subdirectory(tests)

Modified tests/CMakeLists.txt

cmake_minimum_required(VERSION 3.12)

project(nihtest
        VERSION 0.1.0
        DESCRIPTION "NiH testing framework"
        HOMEPAGE_URL "https://github.com/nih-at/nihtest"
        LANGUAGES C)

set(TEST_PROGRAMS
  can-preload
  comparator
  false
  true
  echo
  cat
  getenv
  file
)

foreach(PROGRAM ${TEST_PROGRAMS})
  add_executable(nihtest-${PROGRAM} ${PROGRAM}.c)
endforeach()

ADD_LIBRARY(ineffective-remove MODULE ineffective-remove.c)
TARGET_LINK_LIBRARIES(ineffective-remove ${CMAKE_DL_LIBS})

# Tests for helper programs
add_test(NAME true COMMAND nihtest-true)
add_test(NAME false COMMAND nihtest-false)
set_tests_properties(false PROPERTIES WILL_FAIL TRUE)
add_test(NAME echo COMMAND nihtest-echo "This test was successful.")
set_tests_properties(echo PROPERTIES PASS_REGULAR_EXPRESSION "This test was successful.")
add_test(NAME cat COMMAND nihtest-cat ${CMAKE_CURRENT_SOURCE_DIR}/success.txt)
set_tests_properties(cat PROPERTIES PASS_REGULAR_EXPRESSION "This is a successful test.")
add_test(NAME getenv-defined COMMAND nihtest-getenv "DEFINED")
set_tests_properties(getenv-defined PROPERTIES ENVIRONMENT DEFINED="This is a successful test.")
set_tests_properties(getenv-defined PROPERTIES PASS_REGULAR_EXPRESSION "DEFINED=\"This is a successful test.\"")
add_test(NAME getenv-undefined COMMAND nihtest-getenv "UNDEFINED")
set_tests_properties(getenv-undefined PROPERTIES PASS_REGULAR_EXPRESSION "UNDEFINED=<null>")

file(GLOB TESTS ${CMAKE_CURRENT_SOURCE_DIR}/*.test)
foreach(FULL_CASE IN LISTS TESTS)
  get_filename_component(CASE ${FULL_CASE} NAME)
  add_test(NAME ${CASE} COMMAND python ${NIHTEST} --verbose ${CMAKE_CURRENT_SOURCE_DIR}/${CASE}
           WORKING_DIRECTORY /home/user458/python-nihtest/nihtest-1.1.1/cbuild/test_dir )
  set_tests_properties(${CASE} PROPERTIES SKIP_RETURN_CODE 77)
  set_tests_properties(${CASE} PROPERTIES ENVIRONMENT PATH=/home/user458/python-nihtest/nihtest-1.1.1/cbuild/test_dir/bin:$ENV{PATH})
  set_tests_properties(${CASE} PROPERTIES ENVIRONMENT PYTHONPATH=/home/user458/python-nihtest/nihtest-1.1.1/cbuild/test_dir:$ENV{PYTHONPATH})
endforeach()

# for testing the "features" keyword
set(HAVE_TEST_EXISTING_FEATURE 1)
set(HAVE_TEST_MISSING_FEATURE 0)

configure_file(nihtest.conf.in ${CMAKE_CURRENT_BINARY_DIR}/nihtest.conf @ONLY) # for *.tests
configure_file(nihtest-conf.in ${CMAKE_CURRENT_BINARY_DIR}/nihtest-conf @ONLY) # for *.input

enable_testing()

# TODO: testing depends on venv

After copying nihtest.conf and nihtest-conf to /home/user458/python-nihtest/nihtest-1.1.1/cbuild/test_dir the tests fail with messages such as:

Start 45: stdin-file-pass.test
45/50 Test #45: stdin-file-pass.test ................***Failed    0.21 sec
/home/user458/python-nihtest/nihtest-1.1.1/cbuild/test_dir/bin/nihtest: can't find program nihtest

Any suggestions on how this might be overcome?

preload-pass.test fails with glibc

Running the test suite of nihtest 1.5.0 on linux/glibc results in one failed test case, preload-pass.test. This test can cannot possibly work since glibc uses symbol versioning and the preloaded library defines an unversioned symbol:

> nm -D nihtest-file | grep remove
                 U remove@GLIBC_2.2.5
> nm -D libineffective-remove.so | grep remove
0000000000001110 T remove

But remove@GLIBC_2.2.5 in nihtest-file obviously doesn't bind to the unversioned remove from the preloaded library.

nihtest --version does not work

When using nihtest 1.1.0

pip install --user nihtest
nihtest --version

usage: nihtest [-h] [-C FILE] [--keep-broken] [--no-cleanup] [-q] [--setup-only] [-v] [-V] testcase
nihtest: error: the following arguments are required: testcase