Imported Upstream version 1.2.1
[packages/binwalk.git] / bin / binwalk
1 #!/usr/bin/env python
2
3 import sys
4 import os.path
5 import binwalk
6 from threading import Thread
7 from getopt import GetoptError, getopt as GetOpt
8
9 def display_status():
10         global bwalk
11
12         while True:
13                 # Display the current scan progress when the enter key is pressed.
14                 try:
15                         raw_input()
16                         print "Progress: %.2f%% (%d / %d)\n" % (((float(bwalk.total_scanned) / float(bwalk.scan_length)) * 100), bwalk.total_scanned, bwalk.scan_length)
17                 except Exception, e:
18                         pass
19
20 def examples():
21         name = os.path.basename(sys.argv[0])
22
23         print """
24 Scanning firmware for file signatures:
25
26 \t$ %s firmware.bin
27
28 Extracting files from firmware:
29
30 \t$ %s -Me firmware.bin
31
32 Scanning firmware for executable code:
33
34 \t$ %s -A firmware.bin
35
36 Performing a firmware strings analysis:
37
38 \t$ %s -S firmware.bin
39
40 Performing a firmware entropy analysis:
41
42 \t$ %s -E firmware.bin
43
44 Display identified file signatures on entropy graph:
45
46 \t$ %s -EB firmware.bin
47
48 See http://code.google.com/p/binwalk/wiki/TableOfContents for more.
49 """ % (name, name, name, name, name, name)
50         sys.exit(0)
51
52 def usage(fd):
53         fd.write("\n")
54         
55         fd.write("Binwalk v%s\n" % binwalk.Config.VERSION)
56         fd.write("Craig Heffner, http://www.devttys0.com\n")
57         fd.write("\n")
58         
59         fd.write("Usage: %s [OPTIONS] [FILE1] [FILE2] [FILE3] ...\n" % os.path.basename(sys.argv[0]))
60         fd.write("\n")
61         
62         fd.write("Signature Analysis:\n")
63         fd.write("\t-B, --binwalk                 Perform a file signature scan (default)\n")
64         fd.write("\t-R, --raw-bytes=<string>      Search for a custom signature\n")
65         fd.write("\t-A, --opcodes                 Scan for executable code signatures\n")
66         fd.write("\t-C, --cast                    Cast file contents as various data types\n")
67         fd.write("\t-m, --magic=<file>            Specify an alternate magic file to use\n")
68         fd.write("\t-x, --exclude=<filter>        Exclude matches that have <filter> in their description\n")
69         fd.write("\t-y, --include=<filter>        Only search for matches that have <filter> in their description\n")
70         fd.write("\t-I, --show-invalid            Show results marked as invalid\n")
71         fd.write("\t-k, --keep-going              Show all matching results at a given offset, not just the first one\n")
72         fd.write("\t-b, --dumb                    Disable smart signature keywords\n")
73         fd.write("\n")
74
75         fd.write("Strings Analysis:\n")
76         fd.write("\t-S, --strings                 Scan for ASCII strings (may be combined with -B, -R, -A, or -E)\n")
77         fd.write("\t-s, --strlen=<n>              Set the minimum string length to search for (default: 3)\n")
78         fd.write("\n")
79         
80         fd.write("Entropy Analysis:\n")
81         fd.write("\t-E, --entropy                 Plot file entropy (may be combined with -B, -R, -A, or -S)\n")
82         fd.write("\t-K, --block=<int>             Set the block size for entropy analysis\n")
83         fd.write("\t-a, --shannon                 Use the Shannon entropy algorithm\n")
84         fd.write("\t-N, --no-plot                 Do not generate an entropy plot graph\n")
85         fd.write("\t-F, --marker=<offset:name>    Add a marker to the entropy plot graph\n")
86         fd.write("\t-Q, --no-legend               Omit the legend from the entropy plot graph\n")
87         fd.write("\t-J, --save-plot               Save plot as an SVG (implied if multiple files are specified)\n")
88         fd.write("\n")
89
90         fd.write("Extraction Options:\n")
91         fd.write("\t-D, --dd=<type:ext[:cmd]>     Extract <type> signatures, give the files an extension of <ext>, and execute <cmd>\n")
92         fd.write("\t-e, --extract=[file]          Automatically extract known file types; load rules from file, if specified\n")
93         fd.write("\t-M, --matryoshka              Recursively scan extracted files, up to 8 levels deep\n")
94         fd.write("\t-r, --rm                      Cleanup extracted files and zero-size files\n")
95         fd.write("\t-d, --delay                   Delay file extraction for files with known footers\n")
96         fd.write("\n")
97
98         fd.write("Plugin Options:\n")
99         fd.write("\t-X, --disable-plugin=<name>   Disable a plugin by name\n")
100         fd.write("\t-Y, --enable-plugin=<name>    Enable a plugin by name\n")
101         fd.write("\t-p, --disable-plugins         Do not load any binwalk plugins\n")
102         fd.write("\t-L, --list-plugins            List all user and system plugins by name\n")
103         fd.write("\n")
104
105         fd.write("General Options:\n")  
106         fd.write("\t-o, --offset=<int>            Start scan at this file offset\n")
107         fd.write("\t-l, --length=<int>            Number of bytes to scan\n")
108         fd.write("\t-g, --grep=<text>             Grep results for the specified text\n")
109         fd.write("\t-f, --file=<file>             Log results to file\n")
110         fd.write("\t-c, --csv                     Log results to file in csv format\n")
111         fd.write("\t-O, --skip-unopened           Ignore file open errors and process only the files that can be opened\n")
112         fd.write("\t-t, --term                    Format output to fit the terminal window\n")
113         fd.write("\t-q, --quiet                   Supress output to stdout\n")
114         fd.write("\t-v, --verbose                 Be verbose (specify twice for very verbose)\n")
115         fd.write("\t-u, --update                  Update magic signature files\n")
116         fd.write("\t-?, --examples                Show example usage\n")
117         fd.write("\t-h, --help                    Show help output\n")
118         fd.write("\n")
119
120         if fd == sys.stderr:
121                 sys.exit(1)
122         else:
123                 sys.exit(0)
124
125 def main():
126         # The Binwalk class instance must be global so that the display_status thread can access it.
127         global bwalk
128
129         MIN_ARGC = 2
130
131         requested_scans = []    
132         offset = 0
133         length = 0
134         strlen = 0
135         verbose = 0
136         matryoshka = 1
137         entropy_block = 0
138         failed_open_count = 0
139         quiet = False
140         do_files = False
141         log_file = None
142         do_csv = False
143         save_plot = False
144         show_plot = True
145         show_legend = True
146         entropy_scan = False
147         enable_plugins = True
148         show_invalid = False
149         entropy_algorithm = None
150         format_to_terminal = False
151         custom_signature = None
152         delay_extraction = False
153         extract_rules_file = None
154         ignore_failed_open = False
155         extract_from_config = False
156         cleanup_after_extract = False
157         explicit_signature_scan = False
158         ignore_signature_keywords = False
159         magic_flags = binwalk.magic.MAGIC_NONE
160         markers = []
161         magic_files = []
162         file_opt_list = []
163         target_files = []
164         greps = []
165         excludes = []
166         searches = []
167         extracts = []
168         options = []
169         arguments = []
170         plugin_whitelist = []
171         plugin_blacklist = []
172
173         config = binwalk.Config()
174
175         short_options = "AaBbCcdEehIJkLMNnOPpQqrStuv?D:F:f:g:K:o:l:m:R:s:X:x:Y:y:"
176         long_options = [
177                         "rm",
178                         "help",
179                         "examples",
180                         "quiet", 
181                         "csv",
182                         "verbose",
183                         "opcodes",
184                         "cast",
185                         "update",
186                         "binwalk", 
187                         "keep-going",
188                         "show-invalid",
189                         "profile",
190                         "delay",
191                         "skip-unopened",
192                         "term",
193                         "tim",
194                         "dumb",
195                         "entropy",
196                         "shannon",
197                         "save-plot",
198                         "no-plot",
199                         "no-legend", 
200                         "matryoshka",
201                         "strings",
202                         "list-plugins",
203                         "disable-plugins",
204                         "disable-plugin=",
205                         "enable-plugin=",
206                         "marker=",
207                         "strlen=",
208                         "file=", 
209                         "block=",
210                         "offset=", 
211                         "length=", 
212                         "exclude=",
213                         "include=",
214                         "search=",
215                         "extract=",
216                         "dd=",
217                         "grep=",
218                         "magic=",
219                         "raw-bytes=",
220         ]
221
222         # Require at least one argument (the target file)
223         if len(sys.argv) < MIN_ARGC:
224                 usage(sys.stderr)
225
226         try:
227                 opts, args = GetOpt(sys.argv[1:], short_options, long_options)
228         except GetoptError, e:
229                 sys.stderr.write("%s\n" % str(e))
230                 usage(sys.stderr)
231
232         for opt, arg in opts:
233                 if opt in ("-h", "--help"):
234                         usage(sys.stdout)
235                 elif opt in ("-?", "--examples"):
236                         examples()
237                 elif opt in ("-d", "--delay"):
238                         delay_extraction = True
239                 elif opt in ("-f", "--file"):
240                         log_file = arg
241                 elif opt in ("-c", "--csv"):
242                         do_csv = True
243                 elif opt in ("-q", "--quiet"):
244                         quiet = True
245                 elif opt in ("-s", "--strlen"):
246                         strlen = binwalk.common.str2int(arg)
247                 elif opt in ("-Q", "--no-legend"):
248                         show_legend = False
249                 elif opt in ("-J", "--save-plot"):
250                         save_plot = True
251                 elif opt in ("-E", "--entropy"):
252                         requested_scans.append(binwalk.Binwalk.ENTROPY)
253                 elif opt in ("-a", "--shannon"):
254                         entropy_algorithm = 'shannon'
255                 elif opt in("-t", "--term", "--tim"):
256                         format_to_terminal = True
257                 elif opt in("-p", "--disable-plugins"):
258                         enable_plugins = False
259                 elif opt in ("-b", "--dumb"):
260                         ignore_signature_keywords = True
261                 elif opt in ("-v", "--verbose"):
262                         verbose += 1
263                 elif opt in ("-N", "--no-plot"):
264                         show_plot = False
265                 elif opt in ("-S", "--strings"):
266                         requested_scans.append(binwalk.Binwalk.STRINGS)
267                 elif opt in ("-O", "--skip-unopened"):
268                         ignore_failed_open = True
269                 elif opt in ("-o", "--offset"):
270                         offset = binwalk.common.str2int(arg)
271                 elif opt in ("-l", "--length"):
272                         length = binwalk.common.str2int(arg)
273                 elif opt in ("-y", "--search", "--include"):
274                         searches.append(arg)
275                 elif opt in ("-x", "--exclude"):
276                         excludes.append(arg)
277                 elif opt in ("-D", "--dd"):
278                         extracts.append(arg)
279                 elif opt in ("-g", "--grep"):
280                         greps.append(arg)
281                 elif opt in ("-r", "--rm"):
282                         cleanup_after_extract = True
283                 elif opt in ("-m", "--magic"):
284                         magic_files.append(arg)
285                 elif opt in ("-k", "--keep-going"):
286                         magic_flags |= binwalk.magic.MAGIC_CONTINUE
287                 elif opt in ("-I", "--show-invalid"):
288                         show_invalid = True
289                 elif opt in ("-B", "--binwalk"):
290                         requested_scans.append(binwalk.Binwalk.BINWALK)
291                 elif opt in ("-M", "--matryoshka"):
292                         # Original Zvyozdochkin matrhoska set had 8 dolls. This is a good number.
293                         matryoshka = 8
294                 elif opt in ("-K", "--block"):
295                         entropy_block = binwalk.common.str2int(arg)
296                 elif opt in ("-X", "--disable-plugin"):
297                         plugin_blacklist.append(arg)
298                 elif opt in ("-Y", "--enable-plugin"):
299                         plugin_whitelist.append(arg)
300
301                 elif opt in ("-F", "--marker"):
302                         if ':' in arg:
303                                 (location, description) = arg.split(':', 1)
304                                 location = int(location)
305                                 markers.append((location, [{'description' : description, 'offset' : location}]))
306                 elif opt in("-L", "--list-plugins"):
307                         # List all user and system plugins, then exit
308                         print ''
309                         print 'NAME             TYPE       ENABLED    DESCRIPTION'
310                         print '-' * 115
311                         with binwalk.Binwalk() as bw:
312                                 for (key, info) in binwalk.plugins.Plugins(bw).list_plugins().iteritems():
313                                         for module_name in info['modules']:
314                                                 print '%-16s %-10s %-10s %s' % (module_name, key, info['enabled'][module_name], info['descriptions'][module_name])
315                         print ''
316                         sys.exit(1)
317                 elif opt in ("-e", "--extract"):
318                         # If a file path was specified, use that as the extraction rules file
319                         if arg:
320                                 extract_from_config = False
321                                 extract_rules_file = arg
322                         # Else, use the default rules file
323                         else:
324                                 extract_from_config = True
325                 elif opt in ("-A", "--opcodes"):
326                         requested_scans.append(binwalk.Binwalk.BINARCH)
327                         # Load user file first so its signatures take precedence
328                         magic_files.append(config.paths['user'][config.BINARCH_MAGIC_FILE])
329                         magic_files.append(config.paths['system'][config.BINARCH_MAGIC_FILE])
330                 elif opt in ("-C", "--cast"):
331                         requested_scans.append(binwalk.Binwalk.BINCAST)
332                         # Don't stop at the first match (everything matches everything in this scan)
333                         magic_flags |= binwalk.magic.MAGIC_CONTINUE
334                         # Load user file first so its signatures take precedence
335                         magic_files.append(config.paths['user'][config.BINCAST_MAGIC_FILE])
336                         magic_files.append(config.paths['system'][config.BINCAST_MAGIC_FILE])
337                 elif opt in ("-R", "--raw-bytes"):
338                         custom_signature = arg
339                         requested_scans.append(binwalk.Binwalk.BINWALK)
340                         explicit_signature_scan = True
341                 elif opt in ("-u", "--update"):
342                         try:
343                                 sys.stdout.write("Updating signatures...")
344                                 sys.stdout.flush()
345
346                                 binwalk.Update().update()
347
348                                 sys.stdout.write("done.\n")
349                                 sys.exit(0)
350                         except Exception, e:
351                                 if 'Permission denied' in str(e):
352                                         sys.stderr.write("failed (permission denied). Check your user permissions, or run the update as root.\n")
353                                 else:
354                                         sys.stderr.write('\n' + str(e) + '\n')
355                                 sys.exit(1)
356                 
357                 # The --profile option is handled prior to calling main()
358                 elif opt not in ('-P', '--profile'):
359                         usage(sys.stderr)
360
361                 # Keep track of the options and arguments.
362                 # This is used later to determine which argv entries are file names.
363                 options.append(opt)
364                 options.append("%s%s" % (opt, arg))
365                 options.append("%s=%s" % (opt, arg))
366                 arguments.append(arg)
367                 
368         # Treat any command line options not processed by getopt as target file paths.
369         for opt in sys.argv[1:]:
370                 if opt not in arguments and opt not in options and not opt.startswith('-'):
371                         file_opt_list.append(opt)
372
373         # Validate the target files listed in target_files
374         for tfile in file_opt_list:
375                 # Ignore directories.
376                 if not os.path.isdir(tfile):
377                         # Make sure we can open the target files
378                         try:
379                                 fd = open(tfile, "rb")
380                                 fd.close()
381                                 target_files.append(tfile)
382                         except Exception, e:
383                                 sys.stdout.write("Cannot open file : %s\n" % str(e))
384                                 failed_open_count += 1
385
386         # Unless -O was specified, don't run the scan unless we are able to scan all specified files
387         if failed_open_count > 0 and not ignore_failed_open:
388                 if failed_open_count > 1:
389                         plural = 's'
390                 else:
391                         plural = ''
392                 sys.stdout.write("Failed to open %d file%s for scanning, quitting...\n" % (failed_open_count, plural))
393                 sys.exit(1)
394
395         # If more than one target file was specified, enable verbose mode; else, there is
396         # nothing in the output to indicate which scan corresponds to which file.
397         if (matryoshka > 1 or len(target_files) > 1):
398                 save_plot = True
399                 if not verbose:
400                         verbose = 1
401         elif len(target_files) == 0:
402                 usage(sys.stderr)
403
404         # Instantiate the Binwalk class
405         bwalk = binwalk.Binwalk(flags=magic_flags, verbose=verbose, log=log_file, quiet=quiet, ignore_smart_keywords=ignore_signature_keywords, load_plugins=enable_plugins)
406
407         # If a custom signature was specified, create a temporary magic file containing the custom signature
408         # and ensure that it is the only magic file that will be loaded when Binwalk.scan() is called.
409         if custom_signature is not None:
410                 magic_files = bwalk.parser.file_from_string(custom_signature)
411
412         # Set any specified filters
413         bwalk.filter.exclude(excludes)
414         bwalk.filter.include(searches)
415         bwalk.filter.grep(filters=greps)
416
417         # Add any specified extract rules
418         bwalk.extractor.add_rule(extracts)
419
420         # If -e was specified, load the default extract rules
421         if extract_from_config:
422                 bwalk.extractor.load_defaults()
423
424         # If --extract was specified, load the specified extraction rules file
425         if extract_rules_file is not None:
426                 bwalk.extractor.load_from_file(extract_rules_file)
427
428         # Set the extractor cleanup value (True to clean up files, False to leave them on disk)
429         bwalk.extractor.cleanup_extracted_files(cleanup_after_extract)
430
431         # Enable delayed extraction, which will prevent supported file types from having trailing data when extracted
432         bwalk.extractor.enable_delayed_extract(delay_extraction)
433
434         # Load the magic file(s)
435         bwalk.load_signatures(magic_files=magic_files)
436
437         # If --term was specified, enable output formatting to terminal
438         if format_to_terminal:
439                 bwalk.display.enable_formatting(True)
440
441         # Enable log file CSV formatting, if specified
442         if do_csv:
443                 bwalk.display.enable_csv()
444
445         # If no scan was explicitly rquested, do a binwalk scan
446         if not requested_scans:
447                 requested_scans.append(binwalk.Binwalk.BINWALK)
448
449         # Sort the scan types to ensure the entropy scan is performed last
450         requested_scans.sort()
451
452         # Everything is set up, let's do a scan
453         try:
454                 results = {}
455
456                 # Start the display_status function as a daemon thread.
457                 t = Thread(target=display_status)
458                 t.setDaemon(True)
459                 t.start()
460                 
461                 for scan_type in requested_scans:
462
463                         if scan_type in [binwalk.Binwalk.BINWALK, binwalk.Binwalk.BINARCH, binwalk.Binwalk.BINCAST]:
464
465                                 # There's no generic way for the binwalk class to know what
466                                 # scan type is being run, since these are all signature scans,
467                                 # just with different magic files. Manually set the scan sub-type
468                                 # here to ensure that plugins can differentiate between the
469                                 # scans being performed.
470                                 bwalk.scan_type = scan_type
471
472                                 r = bwalk.scan(target_files,
473                                                 offset=offset, 
474                                                 length=length, 
475                                                 show_invalid_results=show_invalid, 
476                                                 callback=bwalk.display.results, 
477                                                 start_callback=bwalk.display.header,
478                                                 end_callback=bwalk.display.footer,
479                                                 matryoshka=matryoshka,
480                                                 plugins_whitelist=plugin_whitelist,
481                                                 plugins_blacklist=plugin_blacklist)
482
483                                 bwalk.concatenate_results(results, r)
484
485                         elif scan_type == binwalk.Binwalk.STRINGS:
486
487                                 r = bwalk.analyze_strings(target_files, 
488                                                         length=length, 
489                                                         offset=offset, 
490                                                         n=strlen, 
491                                                         block=entropy_block, 
492                                                         algorithm=entropy_algorithm,
493                                                         load_plugins=enable_plugins, 
494                                                         whitelist=plugin_whitelist, 
495                                                         blacklist=plugin_blacklist)
496                                         
497                                 bwalk.concatenate_results(results, r)
498
499                         elif scan_type == binwalk.Binwalk.ENTROPY:
500
501                                 if not results:
502                                         for target_file in target_files:
503                                                 results[target_file] = []
504                                 else:
505                                         bwalk.display.quiet = True
506                                         bwalk.display.cleanup()
507
508                                 for target_file in results.keys():
509                                         bwalk.concatenate_results(results, {target_file : markers})
510
511                                 bwalk.analyze_entropy(results,
512                                                         offset, 
513                                                         length, 
514                                                         entropy_block, 
515                                                         show_plot, 
516                                                         show_legend, 
517                                                         save_plot,
518                                                         algorithm=entropy_algorithm,
519                                                         load_plugins=enable_plugins,
520                                                         whitelist=plugin_whitelist,
521                                                         blacklist=plugin_blacklist)
522
523         except KeyboardInterrupt:
524                 pass
525 #       except Exception, e:
526 #               print "Unexpected error:", str(e)
527                 
528         bwalk.cleanup()
529
530 try:
531         # Special options for profiling the code. For debug use only.
532         if '--profile' in sys.argv or '-P' in sys.argv:
533                 import cProfile
534                 cProfile.run('main()')
535         else:
536                 main()
537 except KeyboardInterrupt:
538         pass
539
540