First round of updates to support new signatures - warvox - VoIP based wardialing tool, forked from rapid7/warvox.
DIR Log
DIR Files
DIR Refs
DIR README
---
DIR commit 39e637f7729852d500d1037f0b2d6071a3a1f36c
DIR parent 8b1275dfbbc99341bfbacb181a71a75799bf3dee
HTML Author: HD Moore <hd_moore@rapid7.com>
Date: Sat, 23 May 2009 07:41:08 +0000
First round of updates to support new signatures
Diffstat:
M bin/create_sig.rb | 26 +++++++++++++++-----------
M etc/sigs/01.default.rb | 98 +++++++++++++++++++------------
M etc/sigs/99.default.rb | 4 ++--
M lib/warvox/audio/raw.rb | 103 ++++++++++++++++++++++++++++++-
M lib/warvox/jobs/analysis.rb | 73 ++++++++++++++++++++++++-------
M web/app/controllers/analyze_contro… | 2 +-
M web/app/controllers/dial_results_c… | 2 +-
A web/db/migrate/20090522202032_add_… | 9 +++++++++
M web/db/schema.rb | 3 ++-
9 files changed, 248 insertions(+), 72 deletions(-)
---
DIR diff --git a/bin/create_sig.rb b/bin/create_sig.rb
@@ -10,24 +10,28 @@ while File.symlink?(base)
end
$:.unshift(File.join(File.expand_path(File.dirname(base)), '..', 'lib'))
require 'warvox'
+require 'yaml'
#
# Script
#
def usage
- $stderr.puts "#{$0} [warvox.db] [num1] [num2] <fuzz-factor> <db-threshold>"
- exit
+ $stderr.puts "#{$0} [raw-file] <skip-count> <length-count>"
+ exit(1)
end
-inp = ARGV.shift() || usage
-num1 = ARGV.shift() || usage
-num2 = ARGV.shift() || usage
-fuzz = (ARGV.shift() || 100).to_i
-thresh = (ARGV.shift() || 800).to_i
+inp = ARGV.shift() || usage()
+skp = (ARGV.shift() || 0).to_i
+len = (ARGV.shift() || 0).to_i
-info1 = []
-info2 = []
+raw = WarVOX::Audio::Raw.from_file(inp)
+raw.samples = (raw.samples[skp, raw.samples.length]||[]) if skp > 0
+raw.samples = (raw.samples[0, len]||[]) if len > 0
-wdb = WarVOX::DB.new(inp, thresh)
-puts wdb.find_sig(num1, num2, { :fuzz => fuzz }).inspect
+if(raw.samples.length == 0)
+ $stderr.puts "Error: the sample length is too short to create a signature"
+ exit(1)
+end
+
+$stdout.puts raw.to_freq.inspect.gsub(/\s+/,'')
DIR diff --git a/etc/sigs/01.default.rb b/etc/sigs/01.default.rb
@@ -3,13 +3,6 @@
#
#
-# Variables:
-# pks = peak frequencies
-# ppz = top 10 frequencies per sample
-# flow = flow signature
-#
-
-#
# These signatures are used first and catch the majority of common
# systems. If you want to force a different type of detection, add
# your signatures to a file starting with "00." and place it in
@@ -19,42 +12,66 @@
#
-# Look for silence by checking for any significant noise
+# Initialize some local variables out of data
+#
+freq = data[:freq]
+fcnt = data[:fcnt]
+maxf = data[:maxf]
+
+#
+# Look for silence by checking the frequency signature
#
-if(flow.split(/\s+/).grep(/^H,/).length == 0)
- line_type = 'silence'
+if(freq.map{|f| f.length}.inject(:+) == 0)
+ @line_type = 'silence'
break
end
+#
+# Look for silence by checking for a strong frequency in each sample
+#
+scnt = 0
+ecnt = 0
+freq.each do |fsec|
+ scnt += 1
+ if(fsec.length == 0)
+ ecnt += 1
+ next
+ end
+ sump = 0
+ fsec.map {|x| sump += x[1] }
+ savg = sump / fsec.length
+ ecnt += 1 if (savg < 100)
+end
+if(ecnt == scnt)
+ @line_type = 'silence'
+ break
+end
+
+# Store these into data for use later on
+data[:scnt] = scnt
+data[:ecnt] = ecnt
#
-# Summarize detection of a whole bunch of frequencies (used below)
+# Look for modems by detecting a 2100hz answer + 2250hz tone
#
-f_2250 = 0
-f_440 = f_350 = 0
-f_1625 = f_1660 = f_1825 = f_2100 = f_1100 = 0
-f_600 = f_1855 = 0
+if( (fcnt[2100] > 1.0 or fcnt[2230] > 1.0) and fcnt[2250] > 0.5)
+ @line_type = 'modem'
+ break
+end
-pkz.each do |fb|
- fb.each do |f|
- f_2250 += 0.1 if(f[0] > 2240 and f[0] < 2260)
- f_440 += 0.1 if(f[0] > 437 and f[0] < 444)
- f_350 += 0.1 if(f[0] > 345 and f[0] < 355)
- f_1625 += 0.1 if(f[0] > 1620 and f[0] < 1630)
- f_1660 += 0.1 if(f[0] > 1655 and f[0] < 1665)
- f_1825 += 0.1 if(f[0] > 1820 and f[0] < 1830)
- f_1855 += 0.1 if(f[0] > 1850 and f[0] < 1860)
- f_2100 += 0.1 if(f[0] > 2090 and f[0] < 2110)
- f_1100 += 0.1 if(f[0] > 1090 and f[0] < 1110)
- f_600 += 0.1 if(f[0] > 595 and f[0] < 605)
- end
+#
+# Look for modems by detecting a peak frequency of 2250hz
+#
+if(fcnt[2100] > 1.0 and (maxf > 2245.0 and maxf < 2255.0))
+ @line_type = 'modem'
+ break
end
#
-# Look for modems by detecting a 2250hz tone
+# Look for modems by detecting a peak frequency of 3000hz
#
-if(f_2250 > 1.0)
- line_type = 'modem'
+if(fcnt[2100] > 1.0 and (maxf > 2995.0 and maxf < 3005.0))
+ @line_type = 'modem'
break
end
@@ -62,17 +79,22 @@ end
# Look for faxes by checking for a handful of tones (min two)
#
fax_sum = 0
-[ f_1625, f_1660, f_1825, f_2100, f_600, f_1855, f_1100].map{|x| fax_sum += [x,1.0].min }
+[
+ fcnt[1625], fcnt[1660], fcnt[1825], fcnt[2100],
+ fcnt[600], fcnt[1855], fcnt[1100], fcnt[2250],
+ fcnt[2230], fcnt[2220], fcnt[1800], fcnt[2095],
+ fcnt[2105]
+].map{|x| fax_sum += [x,1.0].min }
if(fax_sum >= 2.0)
- line_type = 'fax'
+ @line_type = 'fax'
break
end
#
# Dial tone detection (440hz + 350hz)
#
-if(f_440 > 1.0 and f_350 > 1.0)
- line_type = 'dialtone'
+if(fcnt[440] > 1.0 and fcnt[350] > 1.0)
+ @line_type = 'dialtone'
break
end
@@ -83,10 +105,8 @@ end
# this signature can fail. For non-US numbers, the beep
# is often a different frequency entirely.
#
-f_1000 = 0
-pks.each{|f| f_1000 += 1 if(f[0] > 990 and f[0] < 1010) }
-if(f_1000 > 0)
- line_type = 'voicemail'
+if(fcnt[1000] >= 1.0)
+ @line_type = 'voicemail'
break
end
DIR diff --git a/etc/sigs/99.default.rb b/etc/sigs/99.default.rb
@@ -4,7 +4,7 @@
#
# Fall back to 'voice' if nothing else has been matched
-# This should be last signature file processed
+# This should be the last signature file processed
#
-line_type = 'voice'
+@line_type = 'voice'
DIR diff --git a/lib/warvox/audio/raw.rb b/lib/warvox/audio/raw.rb
@@ -2,7 +2,13 @@ module WarVOX
module Audio
class Raw
-
+ @@kissfft_loaded = false
+ begin
+ require 'kissfft'
+ @@kissfft_loaded = true
+ rescue ::LoadError
+ end
+
require 'zlib'
##
@@ -131,6 +137,101 @@ class Raw
return sig
end
+ def to_freq(opts={})
+
+ if(not @@kissfft_loaded)
+ raise RuntimeError, "The KissFFT module is not availabale, raw.to_freq() failed"
+ end
+
+ freq_cnt = opts[:frequency_count] || 20
+
+ # Perform a DFT on the samples
+ ffts = KissFFT.fftr(8192, 8000, 1, self.samples)
+
+ self.class.fft_to_freq_sig(ffts, freq_cnt)
+ end
+
+ def self.fft_to_freq_sig(ffts, freq_cnt)
+ sig = []
+ ffts.each do |s|
+ res = []
+ maxp = 0
+ maxf = 0
+ s.each do |f|
+ if( f[1] > maxp )
+ maxf,maxp = f
+ end
+
+ if(maxf > 0 and f[1] < maxp and (maxf + 4.5 < f[0]))
+ res << [maxf, maxp]
+ maxf,maxp = [0,0]
+ end
+ end
+
+ sig << res.sort{ |a,b| # sort by signal strength
+ a[1] <=> b[1]
+ }.reverse[0,freq_cnt].sort { |a,b| # take the top 20 and sort by frequency
+ a[0] <=> b[0]
+ }.map {|a| [a[0].round, a[1].round ] } # round to whole numbers
+ end
+
+ sig
+ end
+
+ # Find pattern inside of sample
+ def self.compare_freq_sig(pat, zam, opts)
+
+ fuzz_f = opts[:fuzz_f] || 7
+ fuzz_p = opts[:fuzz_p] || 10
+ final = []
+
+ 0.upto(zam.length - 1) do |si|
+ res = []
+ sam = zam[si, zam.length]
+
+ 0.upto(pat.length - 1) do |pi|
+ diff = []
+ next if not pat[pi]
+ next if pat[pi].length == 0
+ pat[pi].each do |x|
+ next if not sam[pi]
+ next if sam[pi].length == 0
+ sam[pi].each do |y|
+ if(
+ (x[0] - fuzz_f) < y[0] and
+ (x[0] + fuzz_f) > y[0] and
+ (x[1] - fuzz_p) < y[1] and
+ (x[1] + fuzz_p) > y[1]
+ )
+ diff << x
+ break
+ end
+ end
+ end
+ res << diff
+ end
+ next if res.length == 0
+
+ prev = 0
+ rsum = 0
+ ridx = 0
+ res.each_index do |xi|
+ len = res[xi].length
+ if(xi == 0)
+ rsum += (len < 2) ? -40 : +20
+ else
+ rsum += 20 if(prev > 11 and len > 11)
+ rsum += len
+ end
+ prev = len
+ end
+
+ final << [ (rsum / res.length.to_f), res.map {|x| x.length}]
+ end
+
+ final
+ end
+
end
end
end
DIR diff --git a/lib/warvox/jobs/analysis.rb b/lib/warvox/jobs/analysis.rb
@@ -14,6 +14,24 @@ class Analysis < Base
rescue ::LoadError
end
+ class SignalProcessor
+ attr_accessor :line_type
+ attr_accessor :signatures
+ attr_accessor :data
+
+ def initialize
+ @signatures = []
+ @data = {}
+ end
+
+ def proc(str)
+ while(true);
+ eval(str)
+ break
+ end
+ end
+ end
+
def type
'analysis'
end
@@ -113,9 +131,11 @@ class Analysis < Base
#
# Create the signature database
- #
+ #
raw = WarVOX::Audio::Raw.from_file(input)
- flow = raw.to_flow
+ fft = KissFFT.fftr(8192, 8000, 1, raw.samples)
+ freq = WarVOX::Audio::Raw.fft_to_freq_sig(fft, 20)
+ flow = freq.inspect.gsub(/\s+/, '')
fd = File.new("#{bname}.sig", "wb")
fd.write "#{num} #{flow}\n"
fd.close
@@ -141,9 +161,6 @@ class Analysis < Base
# Data files for spectrum plotting
frefile = Tempfile.new("frefile")
- # Perform a DFT on the samples
- fft = KissFFT.fftr(8192, 8000, 1, raw.samples)
-
# Calculate the peak frequencies for the sample
maxf = 0
maxp = 0
@@ -174,9 +191,9 @@ class Analysis < Base
fft.each do |slot|
pks << slot.sort{|a,b| a[1] <=> b[1] }.reverse[0]
pkz << slot.sort{|a,b| a[1] <=> b[1] }.reverse[0..9]
- slot.each do |freq|
- avg[ freq[0] ] ||= 0
- avg[ freq[0] ] += freq[1]
+ slot.each do |f|
+ avg[ f[0] ] ||= 0
+ avg[ f[0] ] += f[1]
end
end
@@ -190,25 +207,49 @@ class Analysis < Base
end
frefile.flush
- # Make a guess as to what kind of phone number we found
- line_type = nil
+ # Count significant frequencies across the sample
+ fcnt = {}
+ 0.step(4000, 5) {|f| fcnt[f] = 0 }
+ pkz.each do |fb|
+ fb.each do |f|
+ fdx = ((f[0] / 5.0).round * 5.0).to_i
+ fcnt[fdx] += 0.1
+ end
+ end
+
+ #
+ # Signature processing
+ #
+
+ sproc = SignalProcessor.new
+ sproc.data =
+ {
+ :raw => raw,
+ :freq => freq,
+ :fcnt => fcnt,
+ :fft => fft,
+ :pks => pks,
+ :pkz => pkz,
+ :maxf => maxf,
+ :maxp => maxp
+ }
WarVOX::Config.signatures_load.each do |sigfile|
begin
str = File.read(sigfile, File.size(sigfile))
- while(true)
- eval(str, binding)
- break
- end
+ sproc.proc(str)
rescue ::Exception => e
$stderr.puts "DEBUG: Caught exception in #{sigfile}: #{e} #{e.backtrace}"
end
- break if line_type
+ break if sproc.line_type
end
# Save the guessed line type
- res[:line_type] = line_type
+ res[:line_type] = sproc.line_type
+ # Save any matched signatures
+ res[:signatures] = sproc.signatures.map{|s| "#{s[0]}:#{s[1]}:#{s[2]}" }.join("\n")
+
# Plot samples to a graph
plotter = Tempfile.new("gnuplot")
DIR diff --git a/web/app/controllers/analyze_controller.rb b/web/app/controllers/analyze_controller.rb
@@ -24,7 +24,7 @@ class AnalyzeController < ApplicationController
@g1 = Ezgraphix::Graphic.new(:c_type => 'col3d', :div_name => 'calls_pie1')
@g1.render_options(:caption => 'Detected Lines by Type', :y_name => 'Lines', :w => 700, :h => 300)
- ltypes = DialResult.find( :all, :select => 'DISTINCT line_type' ).map{|r| r.line_type}
+ ltypes = DialResult.find( :all, :select => 'DISTINCT line_type', :conditions => ["dial_job_id = ?", @job_id] ).map{|r| r.line_type}
res_types = {}
ltypes.each do |k|
DIR diff --git a/web/app/controllers/dial_results_controller.rb b/web/app/controllers/dial_results_controller.rb
@@ -50,7 +50,7 @@ class DialResultsController < ApplicationController
@g1 = Ezgraphix::Graphic.new(:c_type => 'col3d', :div_name => 'calls_pie1')
@g1.render_options(:caption => 'Detected Lines by Type', :y_name => 'Lines', :w => 700, :h => 300)
- ltypes = DialResult.find( :all, :select => 'DISTINCT line_type' ).map{|r| r.line_type}
+ ltypes = DialResult.find( :all, :select => 'DISTINCT line_type', :conditions => ["dial_job_id = ?", @job_id] ).map{|r| r.line_type}
res_types = {}
ltypes.each do |k|
DIR diff --git a/web/db/migrate/20090522202032_add_signatures_to_dial_results.rb b/web/db/migrate/20090522202032_add_signatures_to_dial_results.rb
@@ -0,0 +1,9 @@
+class AddSignaturesToDialResults < ActiveRecord::Migration
+ def self.up
+ add_column :dial_results, :signatures, :string
+ end
+
+ def self.down
+ remove_column :dial_results, :signatures
+ end
+end
DIR diff --git a/web/db/schema.rb b/web/db/schema.rb
@@ -9,7 +9,7 @@
#
# It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20090304014033) do
+ActiveRecord::Schema.define(:version => 20090522202032) do
create_table "dial_jobs", :force => true do |t|
t.string "range"
@@ -44,6 +44,7 @@ ActiveRecord::Schema.define(:version => 20090304014033) do
t.string "sig_data"
t.string "line_type"
t.string "notes"
+ t.string "signatures"
end
create_table "providers", :force => true do |t|