Imported Upstream version 1.2.2-1
[packages/binwalk.git] / bin / binwalk
1 #!/usr/bin/env python
2
3 import sys
4 import os.path
5 import binwalk
6 from threading import Thread
7 from getopt import GetoptError, gnu_getopt as GetOpt
8
9 def display_status():
10         global bwalk
11
12         while bwalk is not None:
13                 # Display the current scan progress when the enter key is pressed.
14                 try:
15                         raw_input()
16                         print "Progress: %.2f%% (%d / %d)\n" % (((float(bwalk.total_scanned) / float(bwalk.scan_length)) * 100), bwalk.total_scanned, bwalk.scan_length)
17                 except Exception, e:
18                         pass
19
20 def examples():
21         name = os.path.basename(sys.argv[0])
22
23         print """
24 Scanning firmware for file signatures:
25
26 \t$ %s firmware.bin
27
28 Extracting files from firmware:
29
30 \t$ %s -Me firmware.bin
31
32 Hueristic compression/encryption analysis:
33
34 \t$ %s -H firmware.bin
35
36 Scanning firmware for executable code:
37
38 \t$ %s -A firmware.bin
39
40 Performing a firmware strings analysis:
41
42 \t$ %s -S firmware.bin
43
44 Performing a firmware entropy analysis:
45
46 \t$ %s -E firmware.bin
47
48 Display identified file signatures on entropy graph:
49
50 \t$ %s -EB firmware.bin
51
52 Diffing multiple files:
53
54 \t$ %s -W firmware1.bin firmware2.bin firmware3.bin
55
56 See http://code.google.com/p/binwalk/wiki/TableOfContents for more.
57 """ % (name, name, name, name, name, name, name, name)
58         sys.exit(0)
59
60 def usage(fd):
61         fd.write("\n")
62         
63         fd.write("Binwalk v%s\n" % binwalk.Config.VERSION)
64         fd.write("Craig Heffner, http://www.devttys0.com\n")
65         fd.write("\n")
66         
67         fd.write("Usage: %s [OPTIONS] [FILE1] [FILE2] [FILE3] ...\n" % os.path.basename(sys.argv[0]))
68         fd.write("\n")
69         
70         fd.write("Signature Analysis:\n")
71         fd.write("\t-B, --binwalk                 Perform a file signature scan (default)\n")
72         fd.write("\t-R, --raw-bytes=<string>      Search for a custom signature\n")
73         fd.write("\t-A, --opcodes                 Scan for executable code signatures\n")
74         fd.write("\t-C, --cast                    Cast file contents as various data types\n")
75         fd.write("\t-m, --magic=<file>            Specify an alternate magic file to use\n")
76         fd.write("\t-x, --exclude=<filter>        Exclude matches that have <filter> in their description\n")
77         fd.write("\t-y, --include=<filter>        Only search for matches that have <filter> in their description\n")
78         fd.write("\t-I, --show-invalid            Show results marked as invalid\n")
79         fd.write("\t-T, --ignore-time-skew        Do not show results that have timestamps more than 1 year in the future\n")
80         fd.write("\t-k, --keep-going              Show all matching results at a given offset, not just the first one\n")
81         fd.write("\t-b, --dumb                    Disable smart signature keywords\n")
82         fd.write("\n")
83
84         fd.write("Strings Analysis:\n")
85         fd.write("\t-S, --strings                 Scan for ASCII strings (may be combined with -B, -R, -A, or -E)\n")
86         fd.write("\t-s, --strlen=<n>              Set the minimum string length to search for (default: 3)\n")
87         fd.write("\n")
88         
89         fd.write("Entropy Analysis:\n")
90         fd.write("\t-E, --entropy                 Plot file entropy (may be combined with -B, -R, -A, or -S)\n")
91         fd.write("\t-H, --heuristic               Identify unknown compression/encryption based on entropy heuristics (implies -E)\n")
92         fd.write("\t-K, --block=<int>             Set the block size for entropy analysis (default: %d)\n" % binwalk.entropy.FileEntropy.DEFAULT_BLOCK_SIZE)
93         fd.write("\t-a, --gzip                    Use gzip compression ratios to measure entropy\n")
94         fd.write("\t-N, --no-plot                 Do not generate an entropy plot graph\n")
95         fd.write("\t-F, --marker=<offset:name>    Add a marker to the entropy plot graph\n")
96         fd.write("\t-Q, --no-legend               Omit the legend from the entropy plot graph\n")
97         fd.write("\t-J, --save-plot               Save plot as an SVG (implied if multiple files are specified)\n")
98         fd.write("\n")
99
100         fd.write("Binary Diffing:\n")
101         fd.write("\t-W, --diff                    Hexdump / diff the specified files\n")
102         fd.write("\t-K, --block=<int>             Number of bytes to display per line (default: %d)\n" % binwalk.hexdiff.HexDiff.DEFAULT_BLOCK_SIZE)
103         fd.write("\t-G, --green                   Only show hex dump lines that contain bytes which were the same in all files\n")
104         fd.write("\t-i, --red                     Only show hex dump lines that contain bytes which were different in all files\n")
105         fd.write("\t-U, --blue                    Only show hex dump lines that contain bytes which were different in some files\n")
106         fd.write("\t-w, --terse                   Diff all files, but only display a hex dump of the first file\n")
107         fd.write("\n")
108
109         fd.write("Extraction Options:\n")
110         fd.write("\t-D, --dd=<type:ext[:cmd]>     Extract <type> signatures, give the files an extension of <ext>, and execute <cmd>\n")
111         fd.write("\t-e, --extract=[file]          Automatically extract known file types; load rules from file, if specified\n")
112         fd.write("\t-M, --matryoshka              Recursively scan extracted files, up to 8 levels deep\n")
113         fd.write("\t-r, --rm                      Cleanup extracted files and zero-size files\n")
114         fd.write("\t-d, --delay                   Delay file extraction for files with known footers\n")
115         fd.write("\n")
116
117         fd.write("Plugin Options:\n")
118         fd.write("\t-X, --disable-plugin=<name>   Disable a plugin by name\n")
119         fd.write("\t-Y, --enable-plugin=<name>    Enable a plugin by name\n")
120         fd.write("\t-p, --disable-plugins         Do not load any binwalk plugins\n")
121         fd.write("\t-L, --list-plugins            List all user and system plugins by name\n")
122         fd.write("\n")
123
124         fd.write("General Options:\n")  
125         fd.write("\t-o, --offset=<int>            Start scan at this file offset\n")
126         fd.write("\t-l, --length=<int>            Number of bytes to scan\n")
127         fd.write("\t-g, --grep=<text>             Grep results for the specified text\n")
128         fd.write("\t-f, --file=<file>             Log results to file\n")
129         fd.write("\t-c, --csv                     Log results to file in csv format\n")
130         fd.write("\t-O, --skip-unopened           Ignore file open errors and process only the files that can be opened\n")
131         fd.write("\t-t, --term                    Format output to fit the terminal window\n")
132         fd.write("\t-q, --quiet                   Supress output to stdout\n")
133         fd.write("\t-v, --verbose                 Be verbose (specify twice for very verbose)\n")
134         fd.write("\t-u, --update                  Update magic signature files\n")
135         fd.write("\t-?, --examples                Show example usage\n")
136         fd.write("\t-h, --help                    Show help output\n")
137         fd.write("\n")
138
139         if fd == sys.stderr:
140                 sys.exit(1)
141         else:
142                 sys.exit(0)
143
144 def main():
145         # The Binwalk class instance must be global so that the display_status thread can access it.
146         global bwalk
147
148         MIN_ARGC = 2
149
150         requested_scans = []    
151         offset = 0
152         length = 0
153         strlen = 0
154         verbose = 0
155         matryoshka = 1
156         block_size = 0
157         failed_open_count = 0
158         quiet = False
159         do_comp = False
160         do_files = False
161         log_file = None
162         do_csv = False
163         save_plot = False
164         show_plot = True
165         show_legend = True
166         entropy_scan = False
167         enable_plugins = True
168         show_invalid = False
169         entropy_algorithm = None
170         format_to_terminal = False
171         custom_signature = None
172         delay_extraction = False
173         ignore_time_skew = True
174         extract_rules_file = None
175         ignore_failed_open = False
176         extract_from_config = False
177         show_single_hex_dump = False
178         cleanup_after_extract = False
179         explicit_signature_scan = False
180         ignore_signature_keywords = False
181         magic_flags = binwalk.magic.MAGIC_NONE
182         markers = []
183         magic_files = []
184         file_opt_list = []
185         target_files = []
186         greps = []
187         excludes = []
188         searches = []
189         extracts = []
190         options = []
191         arguments = []
192         plugin_whitelist = []
193         plugin_blacklist = []
194
195         config = binwalk.Config()
196
197         short_options = "AaBbCcdEeGHhIiJkLMNnOPpQqrSTtUuvWw?D:F:f:g:K:o:l:m:R:s:X:x:Y:y:"
198         long_options = [
199                         "rm",
200                         "help",
201                         "green",
202                         "red",
203                         "blue",
204                         "examples",
205                         "quiet", 
206                         "csv",
207                         "verbose",
208                         "opcodes",
209                         "cast",
210                         "update",
211                         "binwalk", 
212                         "keep-going",
213                         "show-invalid",
214                         "ignore-time-skew",
215                         "profile",
216                         "delay",
217                         "skip-unopened",
218                         "term",
219                         "tim",
220                         "terse",
221                         "diff",
222                         "dumb",
223                         "entropy",
224                         "heuristic",
225                         "math",
226                         "gzip",
227                         "save-plot",
228                         "no-plot",
229                         "no-legend", 
230                         "matryoshka",
231                         "strings",
232                         "list-plugins",
233                         "disable-plugins",
234                         "disable-plugin=",
235                         "enable-plugin=",
236                         "marker=",
237                         "strlen=",
238                         "file=", 
239                         "block=",
240                         "offset=", 
241                         "length=", 
242                         "exclude=",
243                         "include=",
244                         "search=",
245                         "extract=",
246                         "dd=",
247                         "grep=",
248                         "magic=",
249                         "raw-bytes=",
250         ]
251
252         # Require at least one argument (the target file)
253         if len(sys.argv) < MIN_ARGC:
254                 usage(sys.stderr)
255
256         try:
257                 opts, args = GetOpt(sys.argv[1:], short_options, long_options)
258         except GetoptError, e:
259                 sys.stderr.write("%s\n" % str(e))
260                 usage(sys.stderr)
261
262         for opt, arg in opts:
263                 if opt in ("-h", "--help"):
264                         usage(sys.stdout)
265                 elif opt in ("-?", "--examples"):
266                         examples()
267                 elif opt in ("-d", "--delay"):
268                         delay_extraction = True
269                 elif opt in ("-f", "--file"):
270                         log_file = arg
271                 elif opt in ("-c", "--csv"):
272                         do_csv = True
273                 elif opt in ("-q", "--quiet"):
274                         quiet = True
275                 elif opt in ("-s", "--strlen"):
276                         strlen = binwalk.common.str2int(arg)
277                 elif opt in ("-Q", "--no-legend"):
278                         show_legend = False
279                 elif opt in ("-J", "--save-plot"):
280                         save_plot = True
281                 elif opt in ("-N", "--no-plot"):
282                         show_plot = False
283                 elif opt in ("-E", "--entropy"):
284                         requested_scans.append(binwalk.Binwalk.ENTROPY)
285                 elif opt in ("-W", "--diff"):
286                         requested_scans.append(binwalk.Binwalk.HEXDIFF)
287                 elif opt in ("-w", "--terse"):
288                         show_single_hex_dump = True
289                 elif opt in ("-a", "--gzip"):
290                         entropy_algorithm = 'gzip'
291                 elif opt in("-t", "--term", "--tim"):
292                         format_to_terminal = True
293                 elif opt in("-p", "--disable-plugins"):
294                         enable_plugins = False
295                 elif opt in ("-b", "--dumb"):
296                         ignore_signature_keywords = True
297                 elif opt in ("-v", "--verbose"):
298                         verbose += 1
299                 elif opt in ("-S", "--strings"):
300                         requested_scans.append(binwalk.Binwalk.STRINGS)
301                 elif opt in ("-O", "--skip-unopened"):
302                         ignore_failed_open = True
303                 elif opt in ("-o", "--offset"):
304                         offset = binwalk.common.str2int(arg)
305                 elif opt in ("-l", "--length"):
306                         length = binwalk.common.str2int(arg)
307                 elif opt in ("-y", "--search", "--include"):
308                         searches.append(arg)
309                 elif opt in ("-x", "--exclude"):
310                         excludes.append(arg)
311                 elif opt in ("-D", "--dd"):
312                         extracts.append(arg)
313                 elif opt in ("-g", "--grep"):
314                         greps.append(arg)
315                 elif opt in ("-G", "--green"):
316                         greps.append("32;")
317                 elif opt in ("-i", "--red"):
318                         greps.append("31;")
319                 elif opt in ("-U", "--blue"):
320                         greps.append("34;")
321                 elif opt in ("-r", "--rm"):
322                         cleanup_after_extract = True
323                 elif opt in ("-m", "--magic"):
324                         magic_files.append(arg)
325                 elif opt in ("-k", "--keep-going"):
326                         magic_flags |= binwalk.magic.MAGIC_CONTINUE
327                 elif opt in ("-I", "--show-invalid"):
328                         show_invalid = True
329                 elif opt in ("-B", "--binwalk"):
330                         requested_scans.append(binwalk.Binwalk.BINWALK)
331                 elif opt in ("-M", "--matryoshka"):
332                         # Original Zvyozdochkin matrhoska set had 8 dolls. This is a good number.
333                         matryoshka = 8
334                 elif opt in ("-K", "--block"):
335                         block_size = binwalk.common.str2int(arg)
336                 elif opt in ("-X", "--disable-plugin"):
337                         plugin_blacklist.append(arg)
338                 elif opt in ("-Y", "--enable-plugin"):
339                         plugin_whitelist.append(arg)
340                 elif opt in ("-T", "--ignore-time-skew"):
341                         ignore_time_skew = False
342
343                 elif opt in ("-H", "--heuristic", "--math"):
344                         do_comp = True
345                         if binwalk.Binwalk.ENTROPY not in requested_scans:
346                                 requested_scans.append(binwalk.Binwalk.ENTROPY)
347                 elif opt in ("-F", "--marker"):
348                         if ':' in arg:
349                                 (location, description) = arg.split(':', 1)
350                                 location = int(location)
351                                 markers.append((location, [{'description' : description, 'offset' : location}]))
352                 elif opt in("-L", "--list-plugins"):
353                         # List all user and system plugins, then exit
354                         print ''
355                         print 'NAME             TYPE       ENABLED    DESCRIPTION'
356                         print '-' * 115
357                         with binwalk.Binwalk() as bw:
358                                 for (key, info) in binwalk.plugins.Plugins(bw).list_plugins().iteritems():
359                                         for module_name in info['modules']:
360                                                 print '%-16s %-10s %-10s %s' % (module_name, key, info['enabled'][module_name], info['descriptions'][module_name])
361                         print ''
362                         sys.exit(1)
363                 elif opt in ("-e", "--extract"):
364                         # If a file path was specified, use that as the extraction rules file
365                         if arg:
366                                 extract_from_config = False
367                                 extract_rules_file = arg
368                         # Else, use the default rules file
369                         else:
370                                 extract_from_config = True
371                 elif opt in ("-A", "--opcodes"):
372                         requested_scans.append(binwalk.Binwalk.BINARCH)
373                         # Load user file first so its signatures take precedence
374                         magic_files.append(config.paths['user'][config.BINARCH_MAGIC_FILE])
375                         magic_files.append(config.paths['system'][config.BINARCH_MAGIC_FILE])
376                 elif opt in ("-C", "--cast"):
377                         requested_scans.append(binwalk.Binwalk.BINCAST)
378                         # Don't stop at the first match (everything matches everything in this scan)
379                         magic_flags |= binwalk.magic.MAGIC_CONTINUE
380                         # Load user file first so its signatures take precedence
381                         magic_files.append(config.paths['user'][config.BINCAST_MAGIC_FILE])
382                         magic_files.append(config.paths['system'][config.BINCAST_MAGIC_FILE])
383                 elif opt in ("-R", "--raw-bytes"):
384                         custom_signature = arg
385                         requested_scans.append(binwalk.Binwalk.BINWALK)
386                         explicit_signature_scan = True
387                 elif opt in ("-u", "--update"):
388                         try:
389                                 sys.stdout.write("Updating signatures...")
390                                 sys.stdout.flush()
391
392                                 binwalk.Update().update()
393
394                                 sys.stdout.write("done.\n")
395                                 sys.exit(0)
396                         except Exception, e:
397                                 if 'Permission denied' in str(e):
398                                         sys.stderr.write("failed (permission denied). Check your user permissions, or run the update as root.\n")
399                                 else:
400                                         sys.stderr.write('\n' + str(e) + '\n')
401                                 sys.exit(1)
402                 
403                 # The --profile option is handled prior to calling main()
404                 elif opt not in ('-P', '--profile'):
405                         usage(sys.stderr)
406
407                 # Keep track of the options and arguments.
408                 # This is used later to determine which argv entries are file names.
409                 options.append(opt)
410                 options.append("%s%s" % (opt, arg))
411                 options.append("%s=%s" % (opt, arg))
412                 arguments.append(arg)
413                 
414         # Treat any command line options not processed by getopt as target file paths.
415         for opt in sys.argv[1:]:
416                 if opt not in arguments and opt not in options and not opt.startswith('-'):
417                         file_opt_list.append(opt)
418
419         # Validate the target files listed in target_files
420         for tfile in file_opt_list:
421                 # Ignore directories.
422                 if not os.path.isdir(tfile):
423                         # Make sure we can open the target files
424                         try:
425                                 fd = open(tfile, "rb")
426                                 fd.close()
427                                 target_files.append(tfile)
428                         except Exception, e:
429                                 sys.stdout.write("Cannot open file : %s\n" % str(e))
430                                 failed_open_count += 1
431
432         # Unless -O was specified, don't run the scan unless we are able to scan all specified files
433         if failed_open_count > 0 and not ignore_failed_open:
434                 if failed_open_count > 1:
435                         plural = 's'
436                 else:
437                         plural = ''
438                 sys.stdout.write("Failed to open %d file%s for scanning, quitting...\n" % (failed_open_count, plural))
439                 sys.exit(1)
440
441         # If more than one target file was specified, enable verbose mode; else, there is
442         # nothing in the output to indicate which scan corresponds to which file.
443         if (matryoshka > 1 or len(target_files) > 1):
444                 save_plot = True
445                 if not verbose:
446                         verbose = 1
447         elif len(target_files) == 0:
448                 usage(sys.stderr)
449
450         # Instantiate the Binwalk class
451         bwalk = binwalk.Binwalk(magic_files=magic_files, flags=magic_flags, verbose=verbose, log=log_file, quiet=quiet, ignore_smart_keywords=ignore_signature_keywords, load_plugins=enable_plugins, ignore_time_skews=ignore_time_skew)
452
453         # If a custom signature was specified, create a temporary magic file containing the custom signature
454         # and ensure that it is the only magic file that will be loaded when Binwalk.scan() is called.
455         if custom_signature is not None:
456                 bwalk.magic_files = [bwalk.parser.file_from_string(custom_signature)]
457
458         # Set any specified filters
459         bwalk.filter.exclude(excludes)
460         bwalk.filter.include(searches)
461         bwalk.filter.grep(filters=greps)
462
463         # Add any specified extract rules
464         bwalk.extractor.add_rule(extracts)
465
466         # If -e was specified, load the default extract rules
467         if extract_from_config:
468                 bwalk.extractor.load_defaults()
469
470         # If --extract was specified, load the specified extraction rules file
471         if extract_rules_file is not None:
472                 bwalk.extractor.load_from_file(extract_rules_file)
473
474         # Set the extractor cleanup value (True to clean up files, False to leave them on disk)
475         bwalk.extractor.cleanup_extracted_files(cleanup_after_extract)
476
477         # Enable delayed extraction, which will prevent supported file types from having trailing data when extracted
478         bwalk.extractor.enable_delayed_extract(delay_extraction)
479
480         # Load the magic file(s)
481         #bwalk.load_signatures(magic_files=magic_files)
482
483         # If --term was specified, enable output formatting to terminal
484         if format_to_terminal:
485                 bwalk.display.enable_formatting(True)
486
487         # Enable log file CSV formatting, if specified
488         if do_csv:
489                 bwalk.display.enable_csv()
490
491         # If no scan was explicitly rquested, do a binwalk scan
492         if not requested_scans:
493                 requested_scans.append(binwalk.Binwalk.BINWALK)
494
495         # Sort the scan types to ensure the entropy scan is performed last
496         requested_scans.sort()
497
498         # Everything is set up, let's do a scan
499         try:
500                 results = {}
501
502                 # Start the display_status function as a daemon thread.
503                 t = Thread(target=display_status)
504                 t.setDaemon(True)
505                 t.start()
506                 
507                 for scan_type in requested_scans:
508
509                         if scan_type in [binwalk.Binwalk.BINWALK, binwalk.Binwalk.BINARCH, binwalk.Binwalk.BINCAST]:
510
511                                 # There's no generic way for the binwalk class to know what
512                                 # scan type is being run, since these are all signature scans,
513                                 # just with different magic files. Manually set the scan sub-type
514                                 # here to ensure that plugins can differentiate between the
515                                 # scans being performed.
516                                 bwalk.scan_type = scan_type
517
518                                 r = bwalk.scan(target_files,
519                                                 offset=offset, 
520                                                 length=length, 
521                                                 show_invalid_results=show_invalid, 
522                                                 callback=bwalk.display.results, 
523                                                 start_callback=bwalk.display.header,
524                                                 end_callback=bwalk.display.footer,
525                                                 matryoshka=matryoshka,
526                                                 plugins_whitelist=plugin_whitelist,
527                                                 plugins_blacklist=plugin_blacklist)
528
529                                 bwalk.concatenate_results(results, r)
530
531                         elif scan_type == binwalk.Binwalk.STRINGS:
532
533                                 r = bwalk.analyze_strings(target_files, 
534                                                         length=length, 
535                                                         offset=offset, 
536                                                         n=strlen, 
537                                                         block=block_size, 
538                                                         load_plugins=enable_plugins, 
539                                                         whitelist=plugin_whitelist, 
540                                                         blacklist=plugin_blacklist)
541                                         
542                                 bwalk.concatenate_results(results, r)
543
544                         elif scan_type == binwalk.Binwalk.COMPRESSION:
545
546                                 r = bwalk.analyze_compression(target_files, offset=offset, length=length)
547                                 bwalk.concatenate_results(results, r)
548
549                         elif scan_type == binwalk.Binwalk.ENTROPY:
550
551                                 if not results:
552                                         for target_file in target_files:
553                                                 results[target_file] = []
554                                 else:
555                                         bwalk.display.quiet = True
556                                         bwalk.display.cleanup()
557
558                                 for target_file in results.keys():
559                                         bwalk.concatenate_results(results, {target_file : markers})
560
561                                 bwalk.analyze_entropy(results,
562                                                         offset, 
563                                                         length, 
564                                                         block_size, 
565                                                         show_plot, 
566                                                         show_legend, 
567                                                         save_plot,
568                                                         algorithm=entropy_algorithm,
569                                                         load_plugins=enable_plugins,
570                                                         whitelist=plugin_whitelist,
571                                                         blacklist=plugin_blacklist,
572                                                         compcheck=do_comp)
573
574                         elif scan_type == binwalk.Binwalk.HEXDIFF:
575                                 
576                                 bwalk.hexdiff(target_files, offset=offset, length=length, block=block_size, first=show_single_hex_dump)
577
578         except KeyboardInterrupt:
579                 pass
580         except IOError:
581                 pass
582         except Exception, e:
583                 print "Unexpected error:", str(e)
584                 
585         bwalk.cleanup()
586
587 try:
588         # Special options for profiling the code. For debug use only.
589         if '--profile' in sys.argv or '-P' in sys.argv:
590                 import cProfile
591                 cProfile.run('main()')
592         else:
593                 main()
594 except KeyboardInterrupt:
595         pass
596
597