import os
import sys
import copy
import glob
import wave
from Riff import *
from pprint import pprint
"""
========================================================================
Convert Stardust Dust Flux Monitor Instrument (DFMI) encounter data to a
WAVE Audio file.
========================================================================
Based on DFMI SoundMaker 2, written by Pasquale DiDonna in Visual Basic.
========================================================================
1) Parse the command line arguments:
1.1) WAVE directory [required]
1.2) Ouput WAVE files name [required]
1.3) Threshold ID for LEFT audio channel [default M1, optional,
one of A1b,A1a,A2b,A2a, M1,M2,M3,M4, m1,m2,m3,m4, N1,N2,N3,N4]
1.4) Threshold ID for RIGHT audio channel [default m1, optional,
one of A1b,A1a,A2b,A2a, M1,M2,M3,M4, m1,m2,m3,m4, N1,N2,N3,N4]
2) Read in sampled WAVE files from WAVE directory (1.1 above):
- tick.wav: contains one click over 100ms
- tick11.wave - tick120.wav: contain 11 to 120 clicks over 1s each
- create a no-sound WAVE sample
2.1) In the Cornell Science Data Center, these WAVE files are in
subdirectory wavs/ relative to where this file, tab2wav.py
resides
3) Read STDIN as the comma-separated-values from a DFMI data file, e.g.
dfmienc_v2_yyyymmdd_hhmmss.tab
one line at a time
4) Check the DFMI SYNC, mode and clock columns:
4.1) DFMI SYNC must be "A5", column 4, or the line is skipped
4.2) DFMI mode must be "NORM", column 5, or the line is skipped
5) Calculate changes across twelve Threshold counters in columns 9-20
from previous line, adding 8- or 16-bit rollover so changes are always
non-negative
6) Compare 1-second DFMI clock, column 3, against previous line's DFMI clock
6.1) When the DFMI clock does not change, accumulate counter changes
within that DFMI clock second
6.2) When the DFMI clock does change, use the accumulated counts from
the previous second, and from the Threshold counter columns
selected (1.3 and 1.4 above), and assign to that second the two
WAVE files with the maximum numbers of clicks less than each of
those counters.
6.2.1) If an accumulated count is greater than 0 and less than
11, the script uses the WAVE data from tick.wav (one
click in 100ms to generate a one-second sample with
the correct number of clicks.
7) When all lines have been read, the concatenate and write out the
samples to the output WAVE file (1.2 above)
========================================================================
Usage:
python tab2wav.py wavs/ out.wav [aLeft[ aRight]] < dfmienc_v2_yyyymmdd_hhmmss.tab
wavs/ => directory containing tick.wav, tick11.wav to tick120.wav
out.wav => output WAVE filepath
aLeft => threshold to use for left speaker: A[12][ab] M[1234] m[1234] N[1234]
aLeft => threshold to use for right speaker: "
e.g. to use all the data from the Tempel 1 encounter
from 04:38:50 to 04:39:30, use this command:
awk -F, \
'$1<"\\"2011-02-15T04:38:50{next}$1>"\\"2011-02-15T04:39:30{next}{print}' \
dfmienc_v2_110215_162959.tab \
| python csv2wav.py wavs out.wav M1 m1 < dfmienc_v1_110215_162959.csv
"""
########################################################################
### Utility routines
########################################################################
bytPerSec=20000
########################################################################
### find the key in tickDict that is closest to nInp without being
### greater than nInp. If 0<nInp<11 and nInp is not in tickDict, create
### a 1s Riff from nInp x 100ms in tickDict[-1] and a lot of nulls.
def getRiffN(nInp):
### If nInp is a list or tuple, return the list of keys
if type(nInp) is list or type(nInp) is tuple:
rtn=[]
for n in nInp: rtn += [getRiffN(n)]
return rtn
### Matching key is in tickDict, return it
if nInp in tickDict: return nInp
### Find largest match less than nInp
if nInp>10:
bestN=0
for N in tickDict:
if nInp>N and N>bestN: bestN = N
return bestN
### create matches 1 to 10 from 100ms in tickDict[-1] ...
nPad = max( 0 ,(bytPerSec/nInp) - tickDict[-1].subChunks['data'][0].Length )
riff1Data = tickDict[-1].subChunks['data'][0].Data + chr(0) * nPad
newRiff = emptyRiff.deepcopy()
while newRiff.subChunks['data'][0].Length<bytPerSec:
newRiff.subChunks['data'][0].addData( (riff1Data*(nInp+1))[:bytPerSec] )
tickDict[nInp] = newRiff
return nInp
########################################################################
### Start ...
emptyRiff = Riff(fyleArg=stryle('RIFF' + struct.pack('<l',36) + 'WAVE'
+ 'fmt ' + struct.pack('<l',16) + (chr(0)*16)
+ 'data' + struct.pack('<l',0) + ''
)
)
###emptyRiff.pout(dataShow=4)
########################################################################
### Parse RIFF WAVE files tick.wav and tickNN.wav and tickNNN.wav in
### directory sys.argv[1] into Riff instances. Each one contains
### 100ms or 1s of 10kHz-sampled ### PCM sound with a number (NN or NNN)
### of clicks.
av=sys.argv[1:]
tickDict={}
for tickFn in glob.glob( os.path.join(av.pop(0),'tick*.wav') ):
bn=os.path.basename(tickFn)
lenbn=len(bn)
if len(bn)<8 : continue ### should be impossible, ignore
if len(bn)==8: id=-1 ### tick.wav, 100ms one click
elif lenbn==9: continue ### tick1.wav or tickN.wav, ignore
elif lenbn>11: continue ### tickNNNN*.wav, too high
else: id=int(bn[4:].split('.')[0]) ### tickNN.wav or tickNNN.wav, 1s, multiple clicks
tickDict[id] = Riff(tickFn)
### Append zeroes if this Riff is shorter than 20,000 bytes
L =tickDict[id].subChunks['data'][0].Length
if id>10 and L<bytPerSec:
tickDict[id].subChunks['data'][0].addData( chr(0)*(bytPerSec-L) )
L =tickDict[id].subChunks['data'][0].Length
########################################################################
### sys.argv[2] is output .wav file
outWavFn = av.pop(0)
########################################################################
### Create RIFF instance of 1s of no sound
### N.B. this will overwrite any tickDict[0] (e.g. tick00.wav)
for id in tickDict:
if id<10: continue
waveRiff=tickDict[id]
tickDict[0] = emptyRiff.deepcopy()
tickDict[0].replaceSubChunk(tickDict[id].subChunks['fmt '][0])
tickDict[0].subChunks['data'][0].addData( chr(0)*tickDict[id].subChunks['data'][0].Length )
break
########################################################################
### Lookup for sys.argv[3:5] to select thresholds
### A[12][ab] - acoustic sensor thresholds
### M[1234] - Large PVDS sensor thresholds
### m[1234] - Small PVDS sensor thresholds
### N[1234] - duplicate acoustic sensor thresholds A[12][ab]
threshDict={}
for i,n in enumerate( 'A1b A1a A2b A2a M1 M2 M3 M4 m1 m2 m3 m4 ,LAST,'.split()): threshDict[n]=i
for i,n in enumerate( 'N1 N2 N3 N4'.split()): threshDict[n]=threshDict['A1b']+i
rollovers = (256,)*4 + (65536,)*8 ### four 8-bit acoustic and eight 16-bit PVDF counters
aLeft = 'M1' ### Default to two channels, M1 and m1
aRight = 'm1'
########################################################################
### Optionally override default channels
### Column offsets of left & right columns, offset from first column
### Calculate where columns are in CSV raw_input() line
### Initialze counters and DFMI clock
### initialize number of output seconds and .Length
if av: aLeft=av.pop(0) ### sys.argv[3] may override aLeft
if av: aRight=av.pop(0) ### sys.argv[3] may override aRight
iLeft,iRight = (threshDict[aLeft],threshDict[aRight])
nCol = threshDict[',LAST,']
firstCol=8
lastCol=firstCol + nCol
newDfmiClk=''
zeroCounters=[0]*nCol
newCounters= copy.copy(zeroCounters)
sumCounters= copy.copy(zeroCounters)
zeroCt = [0]*3
secIds = [zeroCt]*345 ### init buffer at 1k, double as needed
iSeconds = 0
Length = 0
########################################################################
### Loop through lines
try:
while True:
####################################################################
### Split line at commas, skip bad sync and non-NORMal lines
toks=raw_input().split(',')
if toks[3]!='"A5"': continue
if toks[4]!='"NORM"': continue
### Copy last new counters and DFMI clock to old
oldDfmiClk = newDfmiClk
oldCounters=copy.copy(newCounters)
newCounters=eval( '[' + ','.join(toks[firstCol:lastCol]) + ']' )
newDfmiClk=toks[2].strip()
####################################################################
### if this is a new second, save info for old second
if newDfmiClk!=oldDfmiClk:
if not oldDfmiClk: continue ### skip to next line if this is the first second
### Double buffer as needed
if iSeconds==len(secIds): secIds += [zeroCt]*len(secIds)
iL,iR = getRiffN( [sumCounters[iLeft],sumCounters[iRight]] )
dL = min(tickDict[secIds[iSeconds][0]].subChunks['data'][0].Length
,tickDict[secIds[iSeconds][1]].subChunks['data'][0].Length
)
secIds[iSeconds] = [iL,iR,dL]
Length += dL
iSeconds+=1
sumCounters = copy.copy( zeroCounters )
####################################################################
### add counter changes, accounting for rollover
for i,newV in enumerate( newCounters):
sumCounters[i] += (newV - oldCounters[i]) + rollovers[i] * (newV<oldCounters[i])
except:
pass
pprint( dict(Length=Length,iSeconds=iSeconds,lensecIds=len(secIds),oldDfmiClk=oldDfmiClk,newDfmiClk=newDfmiClk) )
###pprint( secIds )
f=wave.open(outWavFn,"wb")
f.setnchannels(2) ### two channels
f.setsampwidth(2) ### 16-bit samples
f.setframerate(10000) ### 10k sample s^-1
f.setcomptype("NONE",'') ### No compression
f.setnframes(Length)
for iL,iR,L in secIds:
dL = tickDict[iL].subChunks['data'][0].Data
dR = tickDict[iR].subChunks['data'][0].Data
for i in range(0,L,2):
f.writeframesraw( dL[i:i+2] + dR[i:i+2] )
f.close()
##pprint( locals() )
"""
"2011-02-15T04:37:53.418","0982212286:147", 1252,"A5","NORM","ENCOUNTER",0,"L", 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289.30,...
"2011-02-15T04:37:54.316","0982212287:121", 1253,"A5","NORM","ENCOUNTER",0,"L", 12, 64, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 289.30,...
"2011-02-15T04:37:54.418","0982212287:147", 1253,"A5","NORM","ENCOUNTER",0,"L", 12, 86, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 289.30,...
"2011-02-15T04:37:55.019","0982212288:045", 1254,"A5","NORM","ENCOUNTER",0,"L", 12, 86, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 289.30,...
"2011-02-15T04:37:55.406","0982212288:144", 1254,"A5","NORM","ENCOUNTER",0,"L", 12, 86, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 289.54,...
"2011-02-15T04:37:56.406","0982212289:144", 1255,"A5","NORM","ENCOUNTER",0,"L", 12, 86, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 289.54,...
"2011-02-15T04:37:57.418","0982212290:147", 1256,"A5","NORM","ENCOUNTER",0,"L", 12, 86, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 289.54,...
"2011-02-15T04:37:58.418","0982212291:147", 1257,"A5","NORM","ENCOUNTER",0,"L", 12, 86, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 289.54,...
"2011-02-15T04:37:59.418","0982212292:147", 1258,"A5","NORM","ENCOUNTER",0,"L", 12, 86, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 289.54,...
"2011-02-15T04:38:00.418","0982212293:147", 1259,"A5","NORM","ENCOUNTER",0,"L", 12, 86, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 289.54,...
"2011-02-15T04:38:01.406","0982212294:144", 1260,"A5","NORM","ENCOUNTER",0,"L", 12, 86, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 289.54,...
"2011-02-15T04:38:01.703","0982212294:220", 1260,"A5","NORM","ENCOUNTER",0,"L", 13,110, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 289.54,...
"2011-02-15T04:38:02.406","0982212295:144", 1261,"A5","NORM","ENCOUNTER",0,"L", 13,134, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 289.54,...
"""