nih-at/nihtest

A testing tool for command line utilities.

nihtest's Issues

Can get a build error since pymodules not explicitly listed

https://stackoverflow.com/questions/72294299/multiple-top-level-packages-discovered-in-a-flat-layout

nihtest --version does not work

When using nihtest 1.1.0

pip install --user nihtest
nihtest --version

usage: nihtest [-h] [-C FILE] [--keep-broken] [--no-cleanup] [-q] [--setup-only] [-v] [-V] testcase
nihtest: error: the following arguments are required: testcase

Running Tests with Local install of nihtest

Am trying to install nihtest in a local folder without using a virtual environment to replicate a packaging build.
Is there some way to modify the configuration to enable the tests to run?

Modified CMakeLists.txt

cmake_minimum_required(VERSION 3.12)

# Also update version in nihtest/__main__.py and pyproject.toml
project(nihtest
        VERSION 1.1.1
        DESCRIPTION "NiH testing framework"
        HOMEPAGE_URL "https://github.com/nih-at/nihtest"
        LANGUAGES C)

enable_testing()

find_package(Python3 REQUIRED COMPONENTS Interpreter)


set(NIHTEST /home/user458/python-nihtest/nihtest-1.1.1/cbuild/test_dir/bin/nihtest)

file(GLOB SOURCES ${CMAKE_SOURCE_DIR}/nihtest/*.py)

add_custom_command(
        OUTPUT ${NIHTEST}
        COMMAND python -m pip install -q --disable-pip-version-check ${CMAKE_SOURCE_DIR} -t test_dir
        DEPENDS ${SOURCES}
)

add_custom_target(mynihtest ALL DEPENDS ${NIHTEST})

# TODO: target for wheel
add_subdirectory(tests)

Modified tests/CMakeLists.txt

cmake_minimum_required(VERSION 3.12)

project(nihtest
        VERSION 0.1.0
        DESCRIPTION "NiH testing framework"
        HOMEPAGE_URL "https://github.com/nih-at/nihtest"
        LANGUAGES C)

set(TEST_PROGRAMS
  can-preload
  comparator
  false
  true
  echo
  cat
  getenv
  file
)

foreach(PROGRAM ${TEST_PROGRAMS})
  add_executable(nihtest-${PROGRAM} ${PROGRAM}.c)
endforeach()

ADD_LIBRARY(ineffective-remove MODULE ineffective-remove.c)
TARGET_LINK_LIBRARIES(ineffective-remove ${CMAKE_DL_LIBS})

# Tests for helper programs
add_test(NAME true COMMAND nihtest-true)
add_test(NAME false COMMAND nihtest-false)
set_tests_properties(false PROPERTIES WILL_FAIL TRUE)
add_test(NAME echo COMMAND nihtest-echo "This test was successful.")
set_tests_properties(echo PROPERTIES PASS_REGULAR_EXPRESSION "This test was successful.")
add_test(NAME cat COMMAND nihtest-cat ${CMAKE_CURRENT_SOURCE_DIR}/success.txt)
set_tests_properties(cat PROPERTIES PASS_REGULAR_EXPRESSION "This is a successful test.")
add_test(NAME getenv-defined COMMAND nihtest-getenv "DEFINED")
set_tests_properties(getenv-defined PROPERTIES ENVIRONMENT DEFINED="This is a successful test.")
set_tests_properties(getenv-defined PROPERTIES PASS_REGULAR_EXPRESSION "DEFINED=\"This is a successful test.\"")
add_test(NAME getenv-undefined COMMAND nihtest-getenv "UNDEFINED")
set_tests_properties(getenv-undefined PROPERTIES PASS_REGULAR_EXPRESSION "UNDEFINED=<null>")

file(GLOB TESTS ${CMAKE_CURRENT_SOURCE_DIR}/*.test)
foreach(FULL_CASE IN LISTS TESTS)
  get_filename_component(CASE ${FULL_CASE} NAME)
  add_test(NAME ${CASE} COMMAND python ${NIHTEST} --verbose ${CMAKE_CURRENT_SOURCE_DIR}/${CASE}
           WORKING_DIRECTORY /home/user458/python-nihtest/nihtest-1.1.1/cbuild/test_dir )
  set_tests_properties(${CASE} PROPERTIES SKIP_RETURN_CODE 77)
  set_tests_properties(${CASE} PROPERTIES ENVIRONMENT PATH=/home/user458/python-nihtest/nihtest-1.1.1/cbuild/test_dir/bin:$ENV{PATH})
  set_tests_properties(${CASE} PROPERTIES ENVIRONMENT PYTHONPATH=/home/user458/python-nihtest/nihtest-1.1.1/cbuild/test_dir:$ENV{PYTHONPATH})
endforeach()

# for testing the "features" keyword
set(HAVE_TEST_EXISTING_FEATURE 1)
set(HAVE_TEST_MISSING_FEATURE 0)

configure_file(nihtest.conf.in ${CMAKE_CURRENT_BINARY_DIR}/nihtest.conf @ONLY) # for *.tests
configure_file(nihtest-conf.in ${CMAKE_CURRENT_BINARY_DIR}/nihtest-conf @ONLY) # for *.input

enable_testing()

# TODO: testing depends on venv

After copying nihtest.conf and nihtest-conf to /home/user458/python-nihtest/nihtest-1.1.1/cbuild/test_dir the tests fail with messages such as:

Start 45: stdin-file-pass.test
45/50 Test #45: stdin-file-pass.test ................***Failed    0.21 sec
/home/user458/python-nihtest/nihtest-1.1.1/cbuild/test_dir/bin/nihtest: can't find program nihtest

Any suggestions on how this might be overcome?

man pages cannot be build from PyPI source

I'm busy packaging nihtest for Fedora. When trying to build the man pages, I discovered that the Makefile is missing from the PyPI tarball:

pushd manpages
make
~/build/BUILD/nihtest-1.1.0/manpages ~/build/BUILD/nihtest-1.1.0
make: *** No targets specified and no makefile found.  Stop.

Could that Makefile be shipped with the PyPI tarball?

I can use the GitHub sources for now, but I'd prefer using the PyPI sources.