From 5f36c99fe17184b3b10edf4d89b8d8f82cedd175 Mon Sep 17 00:00:00 2001 From: Simon Ruderich Date: Mon, 26 Mar 2012 02:19:34 +0200 Subject: [PATCH] Use hashes instead of regexps to check file extensions. Hashes are much faster. Additionally this fixes detecting ".c++" files and false positives caused by files with double extensions (e.g. ".cpp.o") which were not correctly ignored. --- bin/blhc | 205 +++++++++++++++++++++++++++++++--------------------- t/logs/bad | 2 + t/logs/c++ | 2 + t/logs/cc | 2 + t/logs/gcc | 2 + t/logs/good | 2 + t/tests.t | 25 ++++++- 7 files changed, 158 insertions(+), 82 deletions(-) diff --git a/bin/blhc b/bin/blhc index daa0298..f4a242c 100755 --- a/bin/blhc +++ b/bin/blhc @@ -37,92 +37,111 @@ my $cc_regex = qr/(?:[a-z0-9_]+-(?:linux-|kfreebsd-)?gnu(?:eabi|eabihf)?-)? # Regex to catch (GCC) compiler warnings. my $warning_regex = qr/^(.+?):([0-9]+):[0-9]+: warning: (.+?) \[(.+?)\]$/; -# Regex for source files which require preprocessing. -my $source_preprocess_compile_cpp_regex = qr/ +# List of source file extensions which require preprocessing. +my @source_preprocess_compile_cpp = ( # C++ - cc | cp | cxx | cpp | CPP | c\+\+ | C + qw( cc cp cxx cpp CPP c++ C ), # Objective-C++ - | mm | M - /x; -my $source_preprocess_compile_regex = qr/ + qw( mm Mr), +); +my @source_preprocess_compile = ( # C - c + qw( c ), # Objective-C - | m + qw( m ), # (Objective-)C++ - | $source_preprocess_compile_cpp_regex + @source_preprocess_compile_cpp, # Fortran - | F | FOR | fpp | FPP | FTN | F90 | F95 | F03 | F08 - /x; -my $source_preprocess_no_compile_regex = qr/ + qw( F FOR fpp FPP FTN F90 F95 F03 F08 ), +); +my @source_preprocess_no_compile = ( # Assembly - s - /x; -my $source_preprocess_regex = qr/ - $source_preprocess_compile_regex - | $source_preprocess_no_compile_regex - /x; -# Regex for source files which don't require preprocessing. -my $source_no_preprocess_compile_cpp_regex = qr/ + qw( s ), +); +my @source_preprocess = ( + @source_preprocess_compile, + @source_preprocess_no_compile, +); +# List of source file extensions which don't require preprocessing. +my @source_no_preprocess_compile_cpp = ( # C++ - ii + qw( ii ), # Objective-C++ - | mii - /x; -my $source_no_preprocess_compile_regex = qr/ + qw( mii ), +); +my @source_no_preprocess_compile = ( # C - i + qw( i ), # (Objective-)C++ - | $source_no_preprocess_compile_cpp_regex + @source_no_preprocess_compile_cpp, # Objective-C - | mi + qw( mi ), # Fortran - | f | for | ftn | f90 | f95 | f03 | f08 - /x; -my $source_no_preprocess_no_compile_regex = qr/ + qw( f for ftn f90 f95 f03 f08 ), +); +my @source_no_preprocess_no_compile = ( # Assembly - S | sx - /x; -my $source_no_preprocess_regex = qr/ - $source_no_preprocess_compile_regex - | $source_no_preprocess_no_compile_regex - /x; -# Regex for header files which require preprocessing. -my $header_preprocess_regex = qr/ + qw( S sx ), +); +my @source_no_preprocess = ( + @source_no_preprocess_compile, + @source_no_preprocess_no_compile, +); +# List of header file extensions which require preprocessing. +my @header_preprocess = ( # C, C++, Objective-C, Objective-C++ - h + qw( h ), # C++ - | hh | H | hp | hxx | hpp | HPP | h\+\+ | tcc - /x; -# Regexps to match files with the given characteristics. -my $file_no_preprocess_regex = qr/ - $cc_regex.+? - \.(?: $source_no_preprocess_regex)\b - /x; -my $file_preprocess_regex = qr/ - $cc_regex.+? - \.(?: $header_preprocess_regex - | $source_preprocess_regex)\b - /x; -my $file_compile_link_regex = qr/ - $cc_regex.+? - \.(?: $source_preprocess_regex - | $source_no_preprocess_regex)\b - /x; -my $file_compile_regex = qr/ - $cc_regex.+? - \.(?: $source_preprocess_compile_regex - | $source_no_preprocess_compile_regex)\b - /x; -my $file_no_compile_regex = qr/ - $cc_regex.+ - \.(?: $source_preprocess_no_compile_regex - | $source_no_preprocess_no_compile_regex)\b - /x; -my $file_compile_cpp_regex = qr/ - $cc_regex.+ - \.(?: $source_preprocess_compile_cpp_regex - | $source_no_preprocess_compile_cpp_regex)\b + qw( hh H hp hxx hpp HPP h++ tcc ), +); + +# Hashes for fast extensions lookup to check if a file falls in one of these +# categories. +my %extensions_no_preprocess = map { $_ => 1 } ( + @source_no_preprocess, +); +my %extensions_preprocess = map { $_ => 1 } ( + @header_preprocess, + @source_preprocess, +); +my %extensions_compile_link = map { $_ => 1 } ( + @source_preprocess, + @source_no_preprocess, +); +my %extensions_compile = map { $_ => 1 } ( + @source_preprocess_compile, + @source_no_preprocess_compile, +); +my %extensions_no_compile = map { $_ => 1 } ( + @source_preprocess_no_compile, + @source_no_preprocess_no_compile, +); +my %extensions_compile_cpp = map { $_ => 1 } ( + @source_preprocess_compile_cpp, + @source_no_preprocess_compile_cpp, +); +my %extension = map { $_ => 1 } ( + @source_no_preprocess, + @source_no_preprocess_compile, + @source_no_preprocess_compile_cpp, + @source_no_preprocess_no_compile, + @header_preprocess, + @source_preprocess, + @source_preprocess_compile, + @source_preprocess_compile_cpp, + @source_preprocess_no_compile, +); + +# Regexp to match file extensions. +my $file_extension_regex = qr/ + \s + \S+ # Filename without extension. + \. + ([^\\.\s]+) # File extension. + (?=\s|\\) # At end of word. Can't use \b because some files have non + # word characters at the end and because \b matches double + # extensions (like .cpp.o). Works always as all lines are + # terminated with "\n". /x; # Expected (hardening) flags. All flags are used as regexps. @@ -303,6 +322,19 @@ sub is_non_verbose_build { return 1; } +sub extension_found { + my ($extensions_ref, @extensions) = @_; + + my $found = 0; + foreach my $extension (@extensions) { + if (exists $extensions_ref->{$extension}) { + $found = 1; + last; + } + } + return $found; +} + # MAIN @@ -562,6 +594,10 @@ for (my $i = 0; $i < scalar @input; $i++) { # Even if it's a verbose build, we might have to skip this line. next if $skip; + # Remove everything until and including the compiler command. Makes checks + # easier and faster. + $line =~ s/^.*?$cc_regex//; + # Skip unnecessary tests when only preprocessing. my $flag_preprocess = 0; @@ -570,7 +606,7 @@ for (my $i = 0; $i < scalar @input; $i++) { my $link = 0; # Preprocess, compile, assemble. - if ($line =~ /$cc_regex.*?\s(-E|-S|-c)\b/) { + if ($line =~ /\s(-E|-S|-c)\b/) { $preprocess = 1; $flag_preprocess = 1 if $1 eq '-E'; $compile = 1 if $1 eq '-S' or $1 eq '-c'; @@ -579,22 +615,28 @@ for (my $i = 0; $i < scalar @input; $i++) { $link = 1; } + # Get all file extensions on this line. + my @extensions = $line =~ /$file_extension_regex/g; + # Ignore all unknown extensions to speedup the search below. + @extensions = grep { exists $extension{$_} } @extensions; + # These file types don't require preprocessing. - if ($line =~ /$file_no_preprocess_regex/) { + if (extension_found(\%extensions_no_preprocess, @extensions)) { $preprocess = 0; } # These file types require preprocessing. - if ($line =~ /$file_preprocess_regex/) { + if (extension_found(\%extensions_preprocess, @extensions)) { $preprocess = 1; } # If there are source files then it's compiling/linking in one step and we # must check both. We only check for source files here, because header # files cause too many false positives. - if (not $flag_preprocess and $line =~ /$file_compile_link_regex/) { + if (not $flag_preprocess + and extension_found(\%extensions_compile_link, @extensions)) { # Assembly files don't need CFLAGS. - if (not $line =~ /$file_compile_regex/ - and $line =~ /$file_no_compile_regex/) { + if (not extension_found(\%extensions_compile, @extensions) + and extension_found(\%extensions_no_compile, @extensions)) { $compile = 0; # But the rest does. } else { @@ -605,7 +647,8 @@ for (my $i = 0; $i < scalar @input; $i++) { # Assume CXXFLAGS are required when a C++ file is specified in the # compiler line. my $compile_cpp = 0; - if ($compile and $line =~ /$file_compile_cpp_regex/) { + if ($compile + and extension_found(\%extensions_compile_cpp, @extensions)) { $compile = 0; $compile_cpp = 1; } @@ -618,7 +661,7 @@ for (my $i = 0; $i < scalar @input; $i++) { and not pic_pie_conflict($line, $harden_pie, \@missing, @cflags_pie) # Assume dpkg-buildflags returns the correct flags. and not $line =~ /`dpkg-buildflags --get CFLAGS`/) { - error_flags('CFLAGS missing', \@missing, \%flag_renames, $line); + error_flags('CFLAGS missing', \@missing, \%flag_renames, $input[$i]); $exit |= 1 << 3; } elsif ($compile_cpp and not all_flags_used($line, \@missing, @cflags) # Libraries linked with -fPIC don't have to (and can't) be linked @@ -626,13 +669,13 @@ for (my $i = 0; $i < scalar @input; $i++) { and not pic_pie_conflict($line, $harden_pie, \@missing, @cflags_pie) # Assume dpkg-buildflags returns the correct flags. and not $line =~ /`dpkg-buildflags --get CXXFLAGS`/) { - error_flags('CXXFLAGS missing', \@missing, \%flag_renames, $line); + error_flags('CXXFLAGS missing', \@missing, \%flag_renames, $input[$i]); $exit |= 1 << 3; } if ($preprocess and not all_flags_used($line, \@missing, @cppflags) # Assume dpkg-buildflags returns the correct flags. and not $line =~ /`dpkg-buildflags --get CPPFLAGS`/) { - error_flags('CPPFLAGS missing', \@missing, \%flag_renames, $line); + error_flags('CPPFLAGS missing', \@missing, \%flag_renames, $input[$i]); $exit |= 1 << 3; } if ($link and not all_flags_used($line, \@missing, @ldflags) @@ -640,7 +683,7 @@ for (my $i = 0; $i < scalar @input; $i++) { and not pic_pie_conflict($line, $harden_pie, \@missing, @ldflags_pie) # Assume dpkg-buildflags returns the correct flags. and not $line =~ /`dpkg-buildflags --get LDFLAGS`/) { - error_flags('LDFLAGS missing', \@missing, \%flag_renames, $line); + error_flags('LDFLAGS missing', \@missing, \%flag_renames, $input[$i]); $exit |= 1 << 3; } } diff --git a/t/logs/bad b/t/logs/bad index 3240885..f77809b 100644 --- a/t/logs/bad +++ b/t/logs/bad @@ -30,3 +30,5 @@ gcc -c -o test test.s gcc -E test.c gcc test.c + +gcc -Wl,-z,relro -o test test-.cpp test-b.cpp.o diff --git a/t/logs/c++ b/t/logs/c++ index 3e988d3..3999dd6 100644 --- a/t/logs/c++ +++ b/t/logs/c++ @@ -21,6 +21,8 @@ c++\ rm test.c++ c++\ test.c++ +c++\ +test.c++ c++-4.6 -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat-security -Werror=format-security -D_FORTIFY_SOURCE=2 -c test-a.cpp c++-4.6 -g -O2 -fstack-protector -Wformat -Wformat-security -Werror=format-security -D_FORTIFY_SOURCE=2 -c test-b.cpp diff --git a/t/logs/cc b/t/logs/cc index 4fbfca6..bf7c601 100644 --- a/t/logs/cc +++ b/t/logs/cc @@ -9,6 +9,8 @@ cc -Wl,-z,defs -o test test-a.o test-b.o test-c.o -ltest rm cc-test.h cc\ test.cc +cc\ +test.cc rm test.cc diff --git a/t/logs/gcc b/t/logs/gcc index 802b6f1..28bcf28 100644 --- a/t/logs/gcc +++ b/t/logs/gcc @@ -11,6 +11,8 @@ gcc-4.6 -Wl,-z,defs -o test test-a.o test-b.o test-c.o -ltest rm gcc-test.h gcc\ test.c +gcc\ +test.c rm test.gcc diff --git a/t/logs/good b/t/logs/good index 82a6d97..74057f5 100644 --- a/t/logs/good +++ b/t/logs/good @@ -29,3 +29,5 @@ gcc -fPIC -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-s gcc -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security -D_FORTIFY_SOURCE=2 -Wl,-z,relro test.c -o test.output gcc -E -D_FORTIFY_SOURCE=2 test.c + +gcc -Wl,-z,relro -o test test.cpp.o diff --git a/t/tests.t b/t/tests.t index 810f6bc..406f4ea 100644 --- a/t/tests.t +++ b/t/tests.t @@ -111,6 +111,8 @@ CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc -E test.c CFLAGS missing (-g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc test.c CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc test.c LDFLAGS missing (-Wl,-z,relro): gcc test.c +CXXFLAGS missing (-g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc -Wl,-z,relro -o test test-.cpp test-b.cpp.o +CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc -Wl,-z,relro -o test test-.cpp test-b.cpp.o '; is_blhc 'bad', '--pie', 8, 'CFLAGS missing (-fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc -g -O2 -c test-a.c @@ -138,6 +140,9 @@ CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc -E test.c CFLAGS missing (-g -O2 -fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc test.c CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc test.c LDFLAGS missing (-fPIE -pie -Wl,-z,relro): gcc test.c +CXXFLAGS missing (-g -O2 -fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc -Wl,-z,relro -o test test-.cpp test-b.cpp.o +CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc -Wl,-z,relro -o test test-.cpp test-b.cpp.o +LDFLAGS missing (-fPIE -pie): gcc -Wl,-z,relro -o test test-.cpp test-b.cpp.o '; is_blhc 'bad', '--bindnow', 8, 'CFLAGS missing (-fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc -g -O2 -c test-a.c @@ -164,6 +169,9 @@ CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc -E test.c CFLAGS missing (-g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc test.c CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc test.c LDFLAGS missing (-Wl,-z,relro -Wl,-z,now): gcc test.c +CXXFLAGS missing (-g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc -Wl,-z,relro -o test test-.cpp test-b.cpp.o +CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc -Wl,-z,relro -o test test-.cpp test-b.cpp.o +LDFLAGS missing (-Wl,-z,now): gcc -Wl,-z,relro -o test test-.cpp test-b.cpp.o '; is_blhc 'bad', '--pie --bindnow', 8, 'CFLAGS missing (-fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc -g -O2 -c test-a.c @@ -191,6 +199,9 @@ CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc -E test.c CFLAGS missing (-g -O2 -fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc test.c CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc test.c LDFLAGS missing (-fPIE -pie -Wl,-z,relro -Wl,-z,now): gcc test.c +CXXFLAGS missing (-g -O2 -fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc -Wl,-z,relro -o test test-.cpp test-b.cpp.o +CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc -Wl,-z,relro -o test test-.cpp test-b.cpp.o +LDFLAGS missing (-fPIE -pie -Wl,-z,now): gcc -Wl,-z,relro -o test test-.cpp test-b.cpp.o '; is_blhc 'bad', '--all', 8, 'CFLAGS missing (-fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc -g -O2 -c test-a.c @@ -218,6 +229,9 @@ CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc -E test.c CFLAGS missing (-g -O2 -fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc test.c CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc test.c LDFLAGS missing (-fPIE -pie -Wl,-z,relro -Wl,-z,now): gcc test.c +CXXFLAGS missing (-g -O2 -fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc -Wl,-z,relro -o test test-.cpp test-b.cpp.o +CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc -Wl,-z,relro -o test test-.cpp test-b.cpp.o +LDFLAGS missing (-fPIE -pie -Wl,-z,now): gcc -Wl,-z,relro -o test test-.cpp test-b.cpp.o '; is_blhc 'bad-cflags', '', 8, @@ -419,6 +433,9 @@ LDFLAGS missing (-fPIE -pie -Wl,-z,relro -Wl,-z,now): cc -Wl,-z,defs -o test tes CXXFLAGS missing (-g -O2 -fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): cc\ test.cc CPPFLAGS missing (-D_FORTIFY_SOURCE=2): cc\ test.cc LDFLAGS missing (-fPIE -pie -Wl,-z,relro -Wl,-z,now): cc\ test.cc +CXXFLAGS missing (-g -O2 -fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): cc\ test.cc +CPPFLAGS missing (-D_FORTIFY_SOURCE=2): cc\ test.cc +LDFLAGS missing (-fPIE -pie -Wl,-z,relro -Wl,-z,now): cc\ test.cc LDFLAGS missing (-fPIE -pie -Wl,-z,now): cc -Wl,-z,defs test-a.o test-b.o test-c.o -ltest -Wl,-z,relro -o test/test-4.2~_4711/test.so test.o '; @@ -434,6 +451,9 @@ LDFLAGS missing (-fPIE -pie -Wl,-z,relro -Wl,-z,now): gcc-4.6 -Wl,-z,defs -o tes CFLAGS missing (-g -O2 -fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc\ test.c CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc\ test.c LDFLAGS missing (-fPIE -pie -Wl,-z,relro -Wl,-z,now): gcc\ test.c +CFLAGS missing (-g -O2 -fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): gcc\ test.c +CPPFLAGS missing (-D_FORTIFY_SOURCE=2): gcc\ test.c +LDFLAGS missing (-fPIE -pie -Wl,-z,relro -Wl,-z,now): gcc\ test.c LDFLAGS missing (-fPIE -pie -Wl,-z,now): gcc-4.6 -Wl,-z,defs test-a.o test-b.o test-c.o -ltest -Wl,-z,relro -o test/test-4.2~_4711/test.so test.o '; @@ -450,9 +470,12 @@ LDFLAGS missing (-fPIE -pie -Wl,-z,now): c++ -Wl,-z,defs test-a.o test-b.o test- CFLAGS missing (-g -O2 -fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): c++\ test.c CPPFLAGS missing (-D_FORTIFY_SOURCE=2): c++\ test.c LDFLAGS missing (-fPIE -pie -Wl,-z,relro -Wl,-z,now): c++\ test.c -CFLAGS missing (-g -O2 -fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): c++\ test.c++ +CXXFLAGS missing (-g -O2 -fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): c++\ test.c++ CPPFLAGS missing (-D_FORTIFY_SOURCE=2): c++\ test.c++ LDFLAGS missing (-fPIE -pie -Wl,-z,relro -Wl,-z,now): c++\ test.c++ +CXXFLAGS missing (-g -O2 -fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -Werror=format-security): c++\ test.c++ +CPPFLAGS missing (-D_FORTIFY_SOURCE=2): c++\ test.c++ +LDFLAGS missing (-fPIE -pie -Wl,-z,relro -Wl,-z,now): c++\ test.c++ CXXFLAGS missing (-fPIE -Wformat): c++-4.6 -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat-security -Werror=format-security -D_FORTIFY_SOURCE=2 -c test-a.cpp CXXFLAGS missing (-fPIE --param=ssp-buffer-size=4): c++-4.6 -g -O2 -fstack-protector -Wformat -Wformat-security -Werror=format-security -D_FORTIFY_SOURCE=2 -c test-b.cpp CXXFLAGS missing (-fPIE -Werror=format-security): c++-4.6 -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wformat-security -D_FORTIFY_SOURCE=2 -c test-c.cpp -- 2.45.2