Skip to content

Commit e3e9d0f

Browse files
magnumrippermagnumripper
committed
configure/make: Improve plugin handling
Move the plugin logic back to Makefile where it belongs, so we can add or remove plugins without re-running configure. Add logic for fast detection of added or removed plugins. See #3566
1 parent 84ea793 commit e3e9d0f

File tree

8 files changed

+40
-245
lines changed

8 files changed

+40
-245
lines changed

‎.gitignore

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
*.sam
1717

1818
*.o
19+
*.a
1920
*.exe
2021
*.dSYM
2122
.DS_Store
@@ -78,7 +79,7 @@ run/zip2john
7879
.deps/
7980

8081
src/.gdbinit
81-
src/mbedtls/aes.a
82+
src/.plugin_fmt_list
8283
src/arch.h
8384
src/all_tests.lst
8485
src/autoconfig-stamp-h
@@ -98,9 +99,6 @@ src/fmt_registers.h
9899
src/generic.h
99100
src/john_build_rule.h
100101
src/memdbg_defines.h
101-
src/secp256k1/secp256k1.a
102-
src/ed25519-donna/ed25519-donna.a
103-
src/poly1305-donna/poly1305-donna.a
104102
src/stamp-h1
105103
src/version.h
106104
src/version.h.new

‎src/Makefile.in

Lines changed: 36 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ TR = tr
3232
SED = @SED@
3333
FIND = @FIND@
3434
NULL = /dev/null
35-
SORT = @SORT@
35+
SORT = LC_ALL=C @SORT@
3636
STRIP = @STRIP@
3737
PEFLAGS = peflags --dynamicbase=true --nxcompat=true
3838

@@ -82,7 +82,11 @@ OPT_NORMAL = @OPT_NORMAL_FLAGS@
8282
OPT_INLINE = @OPT_INLINE_FLAGS@
8383
#
8484

85-
PLUGFORMATS_OBJS = @PLUGFORMATS_OBJS@
85+
PLUGFORMATS_SRCS := $(filter-out opencl_%_fmt_plug.c, $(wildcard *_fmt_plug.c))
86+
PLUGIN_OBJS := $(filter-out opencl_%_plug.c, $(wildcard *_plug.c))
87+
PLUGIN_OBJS := $(PLUGIN_OBJS:.c=.o)
88+
OPENCL_PLUGIN_OBJS := $(wildcard opencl*_plug.c)
89+
OPENCL_PLUGIN_OBJS := $(OPENCL_PLUGIN_OBJS:.c=.o)
8690

8791
JOHN_OBJS = \
8892
jumbo.o john_mpi.o \
@@ -110,7 +114,7 @@ JOHN_OBJS = \
110114
@UNRAR_OBJS@ \
111115
rar2john.o \
112116
zip2john.o pkzip.o \
113-
$(PLUGFORMATS_OBJS) \
117+
$(PLUGIN_OBJS) \
114118
dyna_salt.o dummy.o \
115119
gost.o \
116120
gpu_common.o \
@@ -135,9 +139,7 @@ JOHN_OBJS = \
135139
showformats.o \
136140
color.o
137141

138-
OCL_OBJS = opencl_common.o opencl_dynamic_loader.o opencl_autotune.o bt.o bt_hash_type_64.o bt_hash_type_128.o bt_hash_type_192.o bt_twister.o
139-
140-
OPENCL_PLUGFORMATS_OBJS = @OPENCL_PLUGFORMATS_OBJS@
142+
OPENCL_OBJS = opencl_common.o opencl_dynamic_loader.o opencl_autotune.o bt.o bt_hash_type_64.o bt_hash_type_128.o bt_hash_type_192.o bt_twister.o
141143

142144
ZTEX_OBJS = ztex_descrypt.o ztex_bcrypt.o ztex_sha512crypt.o ztex_drupal7.o ztex_sha256crypt.o ztex_md5crypt.o ztex_phpass.o ztex_common.o
143145

@@ -164,7 +166,8 @@ WITH_OPENCL=@CL_LIBS@
164166
ifdef WITH_OPENCL
165167
CFLAGS += -DHAVE_OPENCL @CL_CFLAGS@
166168
CFLAGS_MAIN += -DHAVE_OPENCL @CL_CFLAGS@
167-
JOHN_OBJS += $(OCL_OBJS) $(OPENCL_PLUGFORMATS_OBJS)
169+
PLUGFORMATS_SRCS += $(wildcard opencl_*_fmt_plug.c)
170+
JOHN_OBJS += $(OPENCL_OBJS) $(OPENCL_PLUGIN_OBJS)
168171

169172
# core OpenCL dependencies
170173
CL_DEVICE_HEADER = ../run/opencl/opencl_device_info.h
@@ -201,7 +204,6 @@ endif
201204
#
202205
#########################################################
203206
default:
204-
$(MAKE) find_version
205207
@$(MAKE) $(PROJ) \
206208
JOHN_OBJS="$(JOHN_OBJS) @CC_ASM_OBJS@"
207209

@@ -252,15 +254,30 @@ x86-sse.o: x86-sse.S arch.h
252254

253255
x86.o: x86.S arch.h
254256

255-
version.h: find_version
257+
# Phony without the side-effects
258+
FORCE:
256259

257-
find_version:
258-
echo "#define JTR_GIT_VERSION $(JTR_GIT_VERSION)" > version.h.new
259-
diff >/dev/null 2>/dev/null version.h.new version.h && $(RM) version.h.new || $(MV) version.h.new version.h
260+
version.h.new: FORCE
261+
echo "#define JTR_GIT_VERSION $(JTR_GIT_VERSION)" > $@
260262

261-
SUBDIRS = mbedtls secp256k1 ed25519-donna poly1305-donna @ZTEX_SUBDIRS@
263+
version.h: version.h.new
264+
cmp -s $@.new $@ && $(RM) $@.new || $(MV) $@.new $@
265+
266+
# Rebuild the list of plugins, but don't bump timestamp unless content changed.
267+
.plugin_fmt_list: FORCE
268+
@echo $(PLUGFORMATS_SRCS) > $@.new
269+
cmp -s $@.new $@ && $(RM) $@.new || $(MV) $@.new $@
270+
271+
# These are SLOW nowadays, lots of plugins.
272+
fmt_externs.h: .plugin_fmt_list
273+
$(CC) -DFMT_EXTERNS_H -E -P $(CFLAGS) $(PLUGFORMATS_SRCS) | grep -F "extern struct fmt_main" | $(SORT) -f > $@
262274

263-
.PHONY: subdirs $(SUBDIRS) find_version
275+
fmt_registers.h: .plugin_fmt_list
276+
$(CC) -DFMT_REGISTERS_H -E -P $(CFLAGS) $(PLUGFORMATS_SRCS) | grep -F "john_register_one" | $(SORT) -f > $@
277+
278+
.PHONY: subdirs $(SUBDIRS)
279+
280+
SUBDIRS = mbedtls secp256k1 ed25519-donna poly1305-donna @ZTEX_SUBDIRS@
264281

265282
subdirs: $(SUBDIRS)
266283

@@ -433,7 +450,7 @@ poly1305-donna/poly1305-donna.a:
433450
../run/tgtsnarf@EXE_EXT@: tgtsnarf.o
434451
$(LD) tgtsnarf.o $(LDFLAGS) @OPENMP_CFLAGS@ -o $@
435452

436-
john.o: john.c autoconfig.h os.h os-autoconf.h jumbo.h arch.h params.h openssl_local_overrides.h misc.h path.h memory.h list.h tty.h signals.h common.h idle.h formats.h dyna_salt.h loader.h logger.h status.h recovery.h options.h getopt.h config.h bench.h fuzz.h charset.h single.h wordlist.h prince.h inc.h mask.h mkv.h mkvlib.h external.h compiler.h batch.h dynamic.h simd-intrinsics.h pseudo_intrinsics.h aligned.h simd-intrinsics-load-flags.h dynamic_compiler.h fake_salts.h listconf.h crc32.h john_mpi.h regex.h unicode.h $(CL_COMMON_HEADER) $(CL_DEVICE_HEADER) john_build_rule.h fmt_externs.h fmt_registers.h subsets.h
453+
john.o: john.c john.h fmt_externs.h fmt_registers.h
437454
$(CC) $(CFLAGS_MAIN) $(OPT_NORMAL) -O1 $*.c
438455

439456
path.o: path.c path.h autoconfig.h arch.h params.h misc.h memory.h
@@ -711,11 +728,11 @@ distclean: clean kernel-cache-clean-john
711728
(cd $$dir && $(MAKE) distclean) \
712729
|| case "$(MFLAGS)" in *k*) fail=yes;; *) exit 1;; esac; \
713730
done && test -z "$$fail"
714-
$(RM) Makefile autoconfig.h config.status config.cache \
715-
config.log autoconfig-stamp-h autoconfig-stamp-h-in stamp-h1
731+
$(RM) Makefile autoconfig.h config.status config.cache arch.h \
732+
config.log autoconfig-stamp-h autoconfig-stamp-h-in stamp-h1 \
733+
fmt_registers.h fmt_externs.h dynamic_big_crypt.c john_build_rule.h \
734+
version.h .plugin_fmt_list
716735
$(LN) Makefile.stub Makefile
717-
$(RM) arch.h
718-
$(RM) fmt_registers.h fmt_externs.h dynamic_big_crypt.c john_build_rule.h version.h version.h.new
719736

720737
strip: default
721738
@echo Stripping executables.

‎src/configure

Lines changed: 0 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -625,9 +625,6 @@ ac_includes_default="\
625625

626626
ac_subst_vars='LTLIBOBJS
627627
HOST_OS
628-
PLUGFORMATS_DEPS
629-
OPENCL_PLUGFORMATS_OBJS
630-
PLUGFORMATS_OBJS
631628
M4_INCLUDES
632629
HAVE_LIBFUZZER
633630
HAVE_FUZZ
@@ -754,7 +751,6 @@ enable_asan
754751
enable_ubsan
755752
enable_ubsantrap
756753
enable_simd
757-
enable_plugin_dependencies
758754
enable_werror
759755
enable_openmp_for_fast_formats
760756
enable_mpi
@@ -1383,9 +1379,6 @@ Optional Features:
13831379
--disable-simd, --enable-simd=foo
13841380
* Build forcing SIMD logic to be disabled, or force
13851381
use of "foo" for SIMD (eg. --enable-simd=altivec)
1386-
--disable-plugin-dependencies
1387-
Do not create best-effort Makefile dependencies for
1388-
plugins
13891382
--enable-werror Treat warnings as errors
13901383
--enable-openmp-for-fast-formats
13911384
Enable OpenMP for fast formats (usually a bad idea -
@@ -3461,13 +3454,6 @@ else
34613454
simd=yes
34623455
fi
34633456

3464-
# Check whether --enable-plugin-dependencies was given.
3465-
if test "${enable_plugin_dependencies+set}" = set; then :
3466-
enableval=$enable_plugin_dependencies; plug_deps=$enableval
3467-
else
3468-
plug_deps=yes
3469-
fi
3470-
34713457

34723458
# Check whether --enable-werror was given.
34733459
if test "${enable_werror+set}" = set; then :
@@ -16559,27 +16545,6 @@ if test x$with_flock != xno; then
1655916545
fi
1656016546

1656116547

16562-
if test x$using_cl = xyes ; then
16563-
{ $as_echo "$as_me:${as_lineno-$LINENO}: creating *_plug.c and OpenCL object rules" >&5
16564-
$as_echo "$as_me: creating *_plug.c and OpenCL object rules" >&6;}
16565-
else
16566-
{ $as_echo "$as_me:${as_lineno-$LINENO}: creating *_plug.c rules" >&5
16567-
$as_echo "$as_me: creating *_plug.c rules" >&6;}
16568-
fi
16569-
if test "`echo *_plug.c`" != "*_plug.c"; then
16570-
PLUGFORMATS_OBJS=`echo *_plug.c | sed 's/opencl[A-Za-z0-9_\-]*\.c //g;s/\.c/.o/g'`
16571-
16572-
OPENCL_PLUGFORMATS_OBJS=`echo opencl*_plug.c | sed 's/\.c/.o/g'`
16573-
16574-
if test "x$plug_deps" = xyes -a "x$PERL" != x ; then
16575-
{ $as_echo "$as_me:${as_lineno-$LINENO}: creating Makefile dependencies" >&5
16576-
$as_echo "$as_me: creating Makefile dependencies" >&6;}
16577-
16578-
PLUGFORMATS_DEPS=`$PERL ./plugin_deps.pl 2>/dev/null *_plug.c`
16579-
16580-
fi
16581-
fi
16582-
1658316548
{ $as_echo "$as_me:${as_lineno-$LINENO}: creating ./john_build_rule.h" >&5
1658416549
$as_echo "$as_me: creating ./john_build_rule.h" >&6;}
1658516550
echo "#define JOHN_BLD \"${host_os} ${CPU_BIT_STR}-bit${using_x32} ${host_cpu} ${SIMD_NAME} AC\"" > john_build_rule.h
@@ -17967,43 +17932,6 @@ fi
1796717932

1796817933
CFLAGS_EXTRA=$CFLAGS_EXTRA_BACKUP
1796917934

17970-
{ $as_echo "$as_me:${as_lineno-$LINENO}: creating ./fmt_externs.h" >&5
17971-
$as_echo "$as_me: creating ./fmt_externs.h" >&6;}
17972-
rm -f fmt_externs.h
17973-
17974-
CFLAGS_EX2=
17975-
if test "x$simd" = xno ; then
17976-
CFLAGS_EX2=-DJOHN_NO_SIMD
17977-
fi
17978-
17979-
if test "`echo *_fmt_plug.c`" != "*_fmt_plug.c"; then
17980-
$CPP -P -DAC_BUILT $CFLAGS_EX2 $CPPFLAGS $CPU_BEST_FLAGS $CFLAGS $HAVE_OPENCL $CFLAGS_EXTRA $OPENMP_CFLAGS $OPENSSL_CFLAGS -DFMT_EXTERNS_H *_fmt_plug.c | LC_ALL=C $GREP "extern struct fmt_main" | LC_ALL=C $SORT -f > fmt_externs.h
17981-
fi
17982-
if test -f fmt_externs.h -a ! -s fmt_externs.h; then
17983-
cp Makefile.stub Makefile
17984-
as_fn_error $? "The build of fmt_externs.h failed. Can not continue!" "$LINENO" 5
17985-
fi
17986-
if test ! -f fmt_externs.h; then
17987-
echo >fmt_externs.h
17988-
fi
17989-
{ $as_echo "$as_me:${as_lineno-$LINENO}: creating ./fmt_registers.h" >&5
17990-
$as_echo "$as_me: creating ./fmt_registers.h" >&6;}
17991-
rm -f fmt_registers.h
17992-
if test "`echo *_fmt_plug.c`" != "*_fmt_plug.c"; then
17993-
$CPP -P -DAC_BUILT $CFLAGS_EX2 $CPPFLAGS $CPU_BEST_FLAGS $CFLAGS $CFLAGS_EXTRA $OPENMP_CFLAGS $OPENSSL_CFLAGS -DFMT_REGISTERS_H *_fmt_plug.c | LC_ALL=C $GREP "john_register_one" | LC_ALL=C $SORT -f > fmt_registers.h
17994-
fi
17995-
if test -f fmt_registers.h -a ! -s fmt_registers.h; then
17996-
cp Makefile.stub Makefile
17997-
as_fn_error $? "The build of fmt_registers.h failed. Can not continue!" "$LINENO" 5
17998-
fi
17999-
if test ! -f fmt_registers.h; then
18000-
echo >fmt_registers.h
18001-
fi
18002-
18003-
if test "`echo opencl_*_fmt_plug.c`" != "opencl_*_fmt_plug.c"; then
18004-
$CPP -P -DAC_BUILT $CFLAGS_EX2 $CPPFLAGS $CPU_BEST_FLAGS $CFLAGS $HAVE_OPENCL $CFLAGS_EXTRA $OPENMP_CFLAGS $OPENSSL_CFLAGS -DFMT_REGISTERS_H opencl_*_fmt_plug.c | LC_ALL=C $GREP "john_register_one" | LC_ALL=C $SORT -f >> fmt_registers.h
18005-
fi
18006-
1800717935

1800817936
if test "x$ac_cv_prog_c_openmp" != x -a "x$enable_openmp" != xno -a "x$ac_cv_prog_c_openmp" != xunsupported ; then
1800917937
if test "x$ompfast" = "xyes"; then

‎src/configure.ac

Lines changed: 0 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@ AC_ARG_ENABLE([asan], [AS_HELP_STRING([--enable-asan], [* Build with AddressSani
6464
AC_ARG_ENABLE([ubsan], [AS_HELP_STRING([--enable-ubsan], [* Build with UndefinedBehaviorSanitizer])], [ubsan=$enableval], [ubsan=no])
6565
AC_ARG_ENABLE([ubsantrap], [AS_HELP_STRING([--enable-ubsantrap], [* Build with UndefinedBehaviorSanitizer, crashes on failure])], [ubsantrap=$enableval], [ubsantrap=no])
6666
AC_ARG_ENABLE([simd], [AS_HELP_STRING([--disable-simd, --enable-simd=foo], [* Build forcing SIMD logic to be disabled, or force use of "foo" for SIMD (eg. --enable-simd=altivec)])], [simd=$enableval], [simd=yes])
67-
AC_ARG_ENABLE([plugin-dependencies], [AS_HELP_STRING([--disable-plugin-dependencies], [Do not create best-effort Makefile dependencies for plugins])], [plug_deps=$enableval], [plug_deps=yes])
6867

6968
AC_ARG_ENABLE([werror], [AS_HELP_STRING([--enable-werror], [Treat warnings as errors])], [werror=$enableval], [werror=no])
7069

@@ -1037,30 +1036,6 @@ dnl AC_DEFINE([DYNAMIC_DISABLED], 1, [Disables Dynamic format completely])
10371036
dnl fi
10381037
dnl fi
10391038

1040-
dnl ======================================================================
1041-
dnl Code to create @PLUGFORMATS_OBJS@ and
1042-
dnl john_buildrule.h moved from Makefile.in to here.
1043-
dnl don't build opencl_*plug.c unless an opencl build
1044-
dnl ======================================================================
1045-
if test x$using_cl = xyes ; then
1046-
AC_MSG_NOTICE([creating *_plug.c and OpenCL object rules])
1047-
else
1048-
AC_MSG_NOTICE([creating *_plug.c rules])
1049-
fi
1050-
if test "`echo *_plug.c`" != "*_plug.c"; then
1051-
AC_SUBST([PLUGFORMATS_OBJS],[`echo *_plug.c | sed 's/opencl[[A-Za-z0-9_\-]]*\.c //g;s/\.c/.o/g'`])
1052-
AC_SUBST([OPENCL_PLUGFORMATS_OBJS],[`echo opencl*_plug.c | sed 's/\.c/.o/g'`])
1053-
if test "x$plug_deps" = xyes -a "x$PERL" != x ; then
1054-
AC_MSG_NOTICE([creating Makefile dependencies])
1055-
1056-
dnl The "plugin_deps.pl" script isn't clever enough to detect when
1057-
dnl #include statements are ifdef'ed out, so we direct the warnings
1058-
dnl to /dev/null. The mbedTLS directory has lots of inactive code
1059-
dnl which otherwise triggering warnings for "missing" files.
1060-
AC_SUBST([PLUGFORMATS_DEPS],[`$PERL ./plugin_deps.pl 2>/dev/null *_plug.c`])
1061-
fi
1062-
fi
1063-
10641039
AC_MSG_NOTICE([creating ./john_build_rule.h])
10651040
echo "#define JOHN_BLD \"${host_os} ${CPU_BIT_STR}-bit${using_x32} ${host_cpu} ${SIMD_NAME} AC\"" > john_build_rule.h
10661041

@@ -1082,50 +1057,6 @@ AC_OUTPUT(,echo "timestamp from configure.in" > autoconfig-stamp-h)
10821057

10831058
CFLAGS_EXTRA=$CFLAGS_EXTRA_BACKUP
10841059

1085-
dnl ======================================================================
1086-
dnl Create fmt_externs.h and fmt_registers.h
1087-
dnl This needs arch.h to be linked so can't happen before
1088-
dnl AC_OUTPUT
1089-
dnl ======================================================================
1090-
AC_MSG_NOTICE([creating ./fmt_externs.h])
1091-
rm -f fmt_externs.h
1092-
1093-
CFLAGS_EX2=
1094-
if test "x$simd" = xno ; then
1095-
CFLAGS_EX2=-DJOHN_NO_SIMD
1096-
fi
1097-
1098-
if test "`echo *_fmt_plug.c`" != "*_fmt_plug.c"; then
1099-
$CPP -P -DAC_BUILT $CFLAGS_EX2 $CPPFLAGS $CPU_BEST_FLAGS $CFLAGS $HAVE_OPENCL $CFLAGS_EXTRA $OPENMP_CFLAGS $OPENSSL_CFLAGS -DFMT_EXTERNS_H *_fmt_plug.c | LC_ALL=C $GREP "extern struct fmt_main" | LC_ALL=C $SORT -f > fmt_externs.h
1100-
fi
1101-
dnl if the file was not created, then bail.
1102-
if test -f fmt_externs.h -a ! -s fmt_externs.h; then
1103-
cp Makefile.stub Makefile
1104-
AC_MSG_ERROR([The build of fmt_externs.h failed. Can not continue!])
1105-
fi
1106-
dnl if file does not exist, then create a blank one (i.e. no *_plug.c files)
1107-
if test ! -f fmt_externs.h; then
1108-
echo >fmt_externs.h
1109-
fi
1110-
AC_MSG_NOTICE([creating ./fmt_registers.h])
1111-
rm -f fmt_registers.h
1112-
if test "`echo *_fmt_plug.c`" != "*_fmt_plug.c"; then
1113-
$CPP -P -DAC_BUILT $CFLAGS_EX2 $CPPFLAGS $CPU_BEST_FLAGS $CFLAGS $CFLAGS_EXTRA $OPENMP_CFLAGS $OPENSSL_CFLAGS -DFMT_REGISTERS_H *_fmt_plug.c | LC_ALL=C $GREP "john_register_one" | LC_ALL=C $SORT -f > fmt_registers.h
1114-
fi
1115-
dnl if the file was not created, then bail.
1116-
if test -f fmt_registers.h -a ! -s fmt_registers.h; then
1117-
cp Makefile.stub Makefile
1118-
AC_MSG_ERROR([The build of fmt_registers.h failed. Can not continue!])
1119-
fi
1120-
dnl if file does not exist, then create a blank one (i.e. no *_plug.c files)
1121-
if test ! -f fmt_registers.h; then
1122-
echo >fmt_registers.h
1123-
fi
1124-
1125-
if test "`echo opencl_*_fmt_plug.c`" != "opencl_*_fmt_plug.c"; then
1126-
$CPP -P -DAC_BUILT $CFLAGS_EX2 $CPPFLAGS $CPU_BEST_FLAGS $CFLAGS $HAVE_OPENCL $CFLAGS_EXTRA $OPENMP_CFLAGS $OPENSSL_CFLAGS -DFMT_REGISTERS_H opencl_*_fmt_plug.c | LC_ALL=C $GREP "john_register_one" | LC_ALL=C $SORT -f >> fmt_registers.h
1127-
fi
1128-
11291060
dnl ======================================================================
11301061
dnl ONLY _COSMETICAL_ OUTPUT STUFF BELOW THIS LINE
11311062
dnl ======================================================================

‎src/opencl_rar_fmt_plug.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ static size_t max_blob_size;
129129
static int salt_single;
130130

131131
#define RAR_OPENCL_FORMAT
132-
#include "rar_common.c"
132+
#include "rar_common.h"
133133

134134
static void release_clobj(void);
135135

0 commit comments

Comments
 (0)