Merge pull request #4514 from mpg/generated-files-cmake

Generated files cmake

CMakeLists.txt

@@ -46,6 +46,11 @@ option(ENABLE_PROGRAMS "Build mbed TLS programs." ON)
 
 option(UNSAFE_BUILD "Allow unsafe builds. These builds ARE NOT SECURE." OFF)
 option(MBEDTLS_FATAL_WARNINGS "Compiler warnings treated as errors" ON)
+if(WIN32)
+    option(GEN_FILES "Generate the auto-generated files as needed" OFF)
+else()
+    option(GEN_FILES "Generate the auto-generated files as needed" ON)
+endif()
 
 string(REGEX MATCH "Clang" CMAKE_COMPILER_IS_CLANG "${CMAKE_C_COMPILER_ID}")
 string(REGEX MATCH "GNU" CMAKE_COMPILER_IS_GNU "${CMAKE_C_COMPILER_ID}")
@@ -135,6 +140,22 @@ function(link_to_source base_name)
     endif()
 endfunction(link_to_source)
 
+# Get the filename without the final extension (i.e. convert "a.b.c" to "a.b")
+function(get_name_without_last_ext dest_var full_name)
+    # Split into a list on '.' (but a cmake list is just a ';'-separated string)
+    string(REPLACE "." ";" ext_parts "${full_name}")
+    # Remove the last item if there are more than one
+    list(LENGTH ext_parts ext_parts_len)
+    if (${ext_parts_len} GREATER "1")
+        math(EXPR ext_parts_last_item "${ext_parts_len} - 1")
+        list(REMOVE_AT ext_parts ${ext_parts_last_item})
+    endif()
+    # Convert back to a string by replacing separators with '.'
+    string(REPLACE ";" "." no_ext_name "${ext_parts}")
+    # Copy into the desired variable
+    set(${dest_var} ${no_ext_name} PARENT_SCOPE)
+endfunction(get_name_without_last_ext)
+
 string(REGEX MATCH "Clang" CMAKE_COMPILER_IS_CLANG "${CMAKE_C_COMPILER_ID}")
 
 include(CheckCCompilerFlag)

README.md

@@ -67,6 +67,7 @@ If you are cross-compiling, you must set the `CC` environment variable to a C co
 Any of the following methods are available to generate the configuration-independent files:
 
 * If not cross-compiling, running `make` with any target, or just `make`, will automatically generate required files.
+* On non-Windows systems, when not cross-compiling, CMake will generate the required files automatically.
 * Run `make generated_files` to generate all the configuration-independent files.
 * On Unix/POSIX systems, run `tests/scripts/check-generated-files.sh -u` to generate all the configuration-independent files.
 * On Windows, run `scripts\make_generated_files.bat` to generate all the configuration-independent files.

library/CMakeLists.txt

@@ -109,6 +109,44 @@ set(src_tls
     ssl_tls13_generic.c
 )
 
+if(GEN_FILES)
+    find_package(Perl REQUIRED)
+
+    file(GLOB error_headers ${CMAKE_CURRENT_SOURCE_DIR}/../include/mbedtls/*.h)
+    add_custom_command(
+        OUTPUT
+            ${CMAKE_CURRENT_BINARY_DIR}/error.c
+        COMMAND
+            ${PERL_EXECUTABLE}
+                ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/generate_errors.pl
+                ${CMAKE_CURRENT_SOURCE_DIR}/../include/mbedtls
+                ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/data_files
+                ${CMAKE_CURRENT_BINARY_DIR}/error.c
+        DEPENDS
+            ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/generate_errors.pl
+            ${error_headers}
+            ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/data_files/error.fmt
+    )
+
+    add_custom_command(
+        OUTPUT
+            ${CMAKE_CURRENT_BINARY_DIR}/version_features.c
+        COMMAND
+            ${PERL_EXECUTABLE}
+                ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/generate_features.pl
+                ${CMAKE_CURRENT_SOURCE_DIR}/../include/mbedtls
+                ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/data_files
+                ${CMAKE_CURRENT_BINARY_DIR}/version_features.c
+        DEPENDS
+            ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/generate_features.pl
+            ${CMAKE_CURRENT_SOURCE_DIR}/../include/mbedtls/mbedtls_config.h
+            ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/data_files/version_features.fmt
+    )
+else()
+    link_to_source(error.c)
+    link_to_source(version_features.c)
+endif()
+
 if(CMAKE_COMPILER_IS_GNUCC)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wmissing-declarations -Wmissing-prototypes")
 endif(CMAKE_COMPILER_IS_GNUCC)

programs/psa/CMakeLists.txt

@@ -4,6 +4,25 @@ set(executables
     psa_constant_names
 )
 
+if(GEN_FILES)
+    add_custom_command(
+        OUTPUT
+            ${CMAKE_CURRENT_BINARY_DIR}/psa_constant_names_generated.c
+        COMMAND
+            ${MBEDTLS_PYTHON_EXECUTABLE}
+                ${CMAKE_CURRENT_SOURCE_DIR}/../../scripts/generate_psa_constants.py
+                ${CMAKE_CURRENT_BINARY_DIR}
+        WORKING_DIRECTORY
+            ${CMAKE_CURRENT_SOURCE_DIR}/../..
+        DEPENDS
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../scripts/generate_psa_constants.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../include/psa/crypto_values.h
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../include/psa/crypto_extra.h
+    )
+else()
+    link_to_source(psa_constant_names_generated.c)
+endif()
+
 foreach(exe IN LISTS executables)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)
     target_link_libraries(${exe} ${mbedcrypto_target})
@@ -11,6 +30,11 @@ foreach(exe IN LISTS executables)
 endforeach()
 
 target_include_directories(psa_constant_names PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
+if(GEN_FILES)
+    add_custom_target(generate_psa_constant_names_generated_c
+        DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/psa_constant_names_generated.c)
+    add_dependencies(psa_constant_names generate_psa_constant_names_generated_c)
+endif()
 
 install(TARGETS ${executables}
         DESTINATION "bin"

programs/ssl/CMakeLists.txt

@@ -18,23 +18,38 @@ set(executables
     ssl_server2
 )
 
+if(GEN_FILES)
+    # Inform CMake that the following file will be generated as part of the build
+    # process, so it doesn't complain that it doesn't exist yet. Starting from
+    # CMake 3.20, this will no longer be necessary as CMake will automatically
+    # propagate this information across the tree, for now it's only visible
+    # inside the same directory, so we need to propagate manually.
+    set_source_files_properties(
+        ${CMAKE_CURRENT_BINARY_DIR}/../test/query_config.c
+        PROPERTIES GENERATED TRUE)
+endif()
+
 foreach(exe IN LISTS executables)
     set(extra_sources "")
     if(exe STREQUAL "ssl_client2" OR exe STREQUAL "ssl_server2")
         list(APPEND extra_sources
-            ${CMAKE_CURRENT_SOURCE_DIR}/../test/query_config.c)
+            ssl_test_lib.c
+            ${CMAKE_CURRENT_SOURCE_DIR}/../test/query_config.h
+            ${CMAKE_CURRENT_BINARY_DIR}/../test/query_config.c)
     endif()
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>
         ${extra_sources})
     target_link_libraries(${exe} ${libs})
     target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../tests/include)
+    if(exe STREQUAL "ssl_client2" OR exe STREQUAL "ssl_server2")
+        if(GEN_FILES)
+            add_dependencies(${exe} generate_query_config_c)
+        endif()
+        target_include_directories(${exe}
+            PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../test)
+    endif()
 endforeach()
 
-set_property(TARGET ssl_client2 APPEND PROPERTY SOURCES
-    ssl_test_lib.c ${CMAKE_CURRENT_SOURCE_DIR}/../test/query_config.c)
-set_property(TARGET ssl_server2 APPEND PROPERTY SOURCES
-    ssl_test_lib.c ${CMAKE_CURRENT_SOURCE_DIR}/../test/query_config.c)
-
 if(THREADS_FOUND)
     add_executable(ssl_pthread_server ssl_pthread_server.c $<TARGET_OBJECTS:mbedtls_test>)
     target_include_directories(ssl_pthread_server PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../tests/include)

programs/test/CMakeLists.txt

@@ -27,15 +27,44 @@ if(TEST_CPP)
     target_link_libraries(cpp_dummy_build ${mbedcrypto_target})
 endif()
 
+if(GEN_FILES)
+    find_package(Perl REQUIRED)
+
+    add_custom_command(
+        OUTPUT
+            ${CMAKE_CURRENT_BINARY_DIR}/query_config.c
+        COMMAND
+            ${PERL}
+                ${CMAKE_CURRENT_SOURCE_DIR}/../../scripts/generate_query_config.pl
+                ${CMAKE_CURRENT_SOURCE_DIR}/../../include/mbedtls/mbedtls_config.h
+                ${CMAKE_CURRENT_SOURCE_DIR}/../../scripts/data_files/query_config.fmt
+                ${CMAKE_CURRENT_BINARY_DIR}/query_config.c
+        DEPENDS
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../scripts/generate_query_config.pl
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../include/mbedtls/mbedtls_config.h
+            ${CMAKE_CURRENT_SOURCE_DIR}/../../scripts/data_files/query_config.fmt
+    )
+    # this file will also be used in another directory, so create a target, see
+    # https://gitlab.kitware.com/cmake/community/-/wikis/FAQ#how-can-i-add-a-dependency-to-a-source-file-which-is-generated-in-a-subdirectory
+    add_custom_target(generate_query_config_c
+        DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/query_config.c)
+else()
+    link_to_source(query_config.c)
+endif()
+
 foreach(exe IN LISTS executables_libs executables_mbedcrypto)
     set(extra_sources "")
     if(exe STREQUAL "query_compile_time_config")
         list(APPEND extra_sources
-            ${CMAKE_CURRENT_SOURCE_DIR}/query_config.c)
+            ${CMAKE_CURRENT_SOURCE_DIR}/query_config.h
+            ${CMAKE_CURRENT_BINARY_DIR}/query_config.c)
     endif()
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>
         ${extra_sources})
     target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../tests/include)
+    if(exe STREQUAL "query_compile_time_config")
+        target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    endif()
 
     # This emulates "if ( ... IN_LIST ... )" which becomes available in CMake 3.3
     list(FIND executables_libs ${exe} exe_index)

scripts/generate_query_config.pl

@@ -14,7 +14,8 @@
 # information is used to automatically generate the body of the query_config()
 # function by using the template in scripts/data_files/query_config.fmt.
 #
-# Usage: ./scripts/generate_query_config.pl without arguments
+# Usage: scripts/generate_query_config.pl without arguments, or
+# generate_query_config.pl config_file template_file output_file
 #
 # Copyright The Mbed TLS Contributors
 # SPDX-License-Identifier: Apache-2.0
@@ -33,15 +34,24 @@
 
 use strict;
 
-my $config_file = "./include/mbedtls/mbedtls_config.h";
+my ($config_file, $query_config_format_file, $query_config_file);
 
-my $query_config_format_file = "./scripts/data_files/query_config.fmt";
+if( @ARGV ) {
-my $query_config_file = "./programs/test/query_config.c";
+    die "Invalid number of arguments - usage: $0 [CONFIG_FILE TEMPLATE_FILE OUTPUT_FILE]" if scalar @ARGV != 3;
+    ($config_file, $query_config_format_file, $query_config_file) = @ARGV;
 
-unless( -f $config_file && -f $query_config_format_file ) {
+    -f $config_file or die "No such file: $config_file";
-    chdir '..' or die;
+    -f $query_config_format_file or die "No such file: $query_config_format_file";
-    -f $config_file && -f $query_config_format_file
+} else {
-      or die "Without arguments, must be run from root or a subdirectory\n";
+    $config_file = "./include/mbedtls/mbedtls_config.h";
+    $query_config_format_file = "./scripts/data_files/query_config.fmt";
+    $query_config_file = "./programs/test/query_config.c";
+
+    unless( -f $config_file && -f $query_config_format_file ) {
+        chdir '..' or die;
+        -f $config_file && -f $query_config_format_file
+          or die "No arguments supplied, must be run from project root or a first-level subdirectory\n";
+    }
 }
 
 # Excluded macros from the generated query_config.c. For example, macros that

tests/CMakeLists.txt

@@ -13,10 +13,48 @@ if(NOT MBEDTLS_PYTHON_EXECUTABLE)
     message(FATAL_ERROR "Cannot build test suites without Python 3")
 endif()
 
-# Enable definition of various functions used throughout the testsuite
+# generated .data files will go there
-# (gethostname, strdup, fileno...) even when compiling with -std=c99. Harmless
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/suites)
-# on non-POSIX platforms.
+
-add_definitions("-D_POSIX_C_SOURCE=200809L")
+# Get base names for generated files (starting at "suites/")
+execute_process(
+    COMMAND
+        ${MBEDTLS_PYTHON_EXECUTABLE}
+        ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_psa_tests.py
+        --list-for-cmake
+        --directory suites
+    WORKING_DIRECTORY
+        ${CMAKE_CURRENT_SOURCE_DIR}/..
+    OUTPUT_VARIABLE
+        base_generated_data_files)
+
+# Derive generated file paths in the build directory
+set(generated_data_files "")
+foreach(file ${base_generated_data_files})
+    list(APPEND generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/${file})
+endforeach()
+
+if(GEN_FILES)
+    add_custom_command(
+        OUTPUT
+            ${generated_data_files}
+        WORKING_DIRECTORY
+            ${CMAKE_CURRENT_SOURCE_DIR}/..
+        COMMAND
+            ${MBEDTLS_PYTHON_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_psa_tests.py
+            --directory ${CMAKE_CURRENT_BINARY_DIR}/suites
+        DEPENDS
+            ${CMAKE_CURRENT_SOURCE_DIR}/../tests/scripts/generate_psa_tests.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_config.h
+            ${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_values.h
+            ${CMAKE_CURRENT_SOURCE_DIR}/../include/psa/crypto_extra.h
+    )
+else()
+    foreach(file ${base_generated_data_files})
+        link_to_source(${file})
+    endforeach()
+endif()
 
 # Test suites caught by SKIP_TEST_SUITES are built but not executed.
 # "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar"
@@ -32,10 +70,52 @@ function(add_test_suite suite_name)
         set(data_name ${suite_name})
     endif()
 
+    # Get the test names of the tests with generated .data files
+    # from the generated_data_files list in parent scope.
+    set(generated_data_names "")
+    foreach(generated_data_file ${generated_data_files})
+        # Get the plain filename
+        get_filename_component(generated_data_name ${generated_data_file} NAME)
+        # Remove the ".data" extension
+        get_name_without_last_ext(generated_data_name ${generated_data_name})
+        # Remove leading "test_suite_"
+        string(SUBSTRING ${generated_data_name} 11 -1 generated_data_name)
+        list(APPEND generated_data_names ${generated_data_name})
+    endforeach()
+
+    if(";${generated_data_names};" MATCHES ";${data_name};")
+        set(data_file
+            ${CMAKE_CURRENT_BINARY_DIR}/suites/test_suite_${data_name}.data)
+    else()
+        set(data_file
+            ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${data_name}.data)
+    endif()
+
     add_custom_command(
-        OUTPUT test_suite_${data_name}.c
+        OUTPUT
-        COMMAND ${MBEDTLS_PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_test_code.py -f ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${suite_name}.function -d ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${data_name}.data -t ${CMAKE_CURRENT_SOURCE_DIR}/suites/main_test.function -p ${CMAKE_CURRENT_SOURCE_DIR}/suites/host_test.function -s ${CMAKE_CURRENT_SOURCE_DIR}/suites --helpers-file ${CMAKE_CURRENT_SOURCE_DIR}/suites/helpers.function -o .
+            # The output filename of generate_test_code.py is derived from the -d
-        DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_test_code.py ${mbedtls_target} ${CMAKE_CURRENT_SOURCE_DIR}/suites/helpers.function ${CMAKE_CURRENT_SOURCE_DIR}/suites/main_test.function ${CMAKE_CURRENT_SOURCE_DIR}/suites/host_test.function ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${suite_name}.function ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${data_name}.data
+            # input argument.
+            test_suite_${data_name}.c
+        COMMAND
+            ${MBEDTLS_PYTHON_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_test_code.py
+            -f ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${suite_name}.function
+            -d ${data_file}
+            -t ${CMAKE_CURRENT_SOURCE_DIR}/suites/main_test.function
+            -p ${CMAKE_CURRENT_SOURCE_DIR}/suites/host_test.function
+            -s ${CMAKE_CURRENT_SOURCE_DIR}/suites
+            --helpers-file ${CMAKE_CURRENT_SOURCE_DIR}/suites/helpers.function
+            -o .
+        DEPENDS
+            ${CMAKE_CURRENT_SOURCE_DIR}/scripts/generate_test_code.py
+            ${CMAKE_CURRENT_SOURCE_DIR}/suites/test_suite_${suite_name}.function
+            ${data_file}
+            ${CMAKE_CURRENT_SOURCE_DIR}/suites/main_test.function
+            ${CMAKE_CURRENT_SOURCE_DIR}/suites/host_test.function
+            ${CMAKE_CURRENT_SOURCE_DIR}/suites/helpers.function
+            ${mbedtls_target}
+        BYPRODUCTS
+            test_suite_${data_name}.datax
     )
 
     add_executable(test_suite_${data_name} test_suite_${data_name}.c $<TARGET_OBJECTS:mbedtls_test>)
@@ -55,6 +135,11 @@ function(add_test_suite suite_name)
     endif()
 endfunction(add_test_suite)
 
+# Enable definition of various functions used throughout the testsuite
+# (gethostname, strdup, fileno...) even when compiling with -std=c99. Harmless
+# on non-POSIX platforms.
+add_definitions("-D_POSIX_C_SOURCE=200809L")
+
 if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function")
 endif(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)
@@ -170,5 +255,4 @@ if (NOT ${CMAKE_CURRENT_BINARY_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR})
     link_to_source(data_files)
     link_to_source(scripts)
     link_to_source(ssl-opt.sh)
-    link_to_source(suites)
 endif()

tests/scripts/all.sh

@@ -292,7 +292,8 @@ cleanup()
            -iname CMakeFiles -exec rm -rf {} \+ -o \
            \( -iname cmake_install.cmake -o \
               -iname CTestTestfile.cmake -o \
-              -iname CMakeCache.txt \) -exec rm -f {} \+
+              -iname CMakeCache.txt -o \
+              -path './cmake/*.cmake' \) -exec rm -f {} \+
     # Recover files overwritten by in-tree CMake builds
     rm -f include/Makefile include/mbedtls/Makefile programs/*/Makefile
 
@@ -998,7 +999,16 @@ component_test_psa_crypto_rsa_no_genprime() {
 
 component_test_ref_configs () {
     msg "test/build: ref-configs (ASan build)" # ~ 6 min 20s
-    CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan .
+    # test-ref-configs works by overwriting mbedtls_config.h; this makes cmake
+    # want to re-generate generated files that depend on it, quite correctly.
+    # However this doesn't work as the generation script expects a specific
+    # format for mbedtls_config.h, which the other files don't follow. Also,
+    # cmake can't know this, but re-generation is actually not necessary as
+    # the generated files only depend on the list of available options, not
+    # whether they're on or off. So, disable cmake's (over-sensitive here)
+    # dependency resolution for generated files and just rely on them being
+    # present (thanks to pre_generate_files) by turning GEN_FILES off.
+    CC=gcc cmake -D GEN_FILES=Off -D CMAKE_BUILD_TYPE:String=Asan .
     tests/scripts/test-ref-configs.pl
 }
 

tests/scripts/generate_psa_tests.py

@@ -699,6 +699,10 @@ def main(args):
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('--list', action='store_true',
                         help='List available targets and exit')
+    parser.add_argument('--list-for-cmake', action='store_true',
+                        help='Print \';\'-separated list of available targets and exit')
+    parser.add_argument('--directory', metavar='DIR',
+                        help='Output directory (default: tests/suites)')
     parser.add_argument('targets', nargs='*', metavar='TARGET',
                         help='Target file to generate (default: all; "-": none)')
     options = parser.parse_args(args)
@@ -708,6 +712,11 @@ def main(args):
         for name in sorted(generator.TARGETS):
             print(generator.filename_for(name))
         return
+    # List in a cmake list format (i.e. ';'-separated)
+    if options.list_for_cmake:
+        print(';'.join(generator.filename_for(name)
+                       for name in sorted(generator.TARGETS)), end='')
+        return
     if options.targets:
         # Allow "-" as a special case so you can run
         # ``generate_psa_tests.py - $targets`` and it works uniformly whether