PDS Note ================================================================================ The Python code fir_filter.py was used by the InSight IFG team to generate the ground 0.2 Hz data (gpt2Hz) from the 2 Hz data. The code is included in the archive as documentation - it will not work with the data files in the archive. Like the other python codes that have been included, the code reads in and writes out IGPP flatfiles (a self describing binary format). The filter coefficient files are included in this file after the code. ================================================================================ README Copyright (c) 2023 Regents of the University of California All Rights Reserved ================================================================================ script usage - Run default parameters (reducing the data rate by a factor of 2 and then again by a factor of 5) do the following % python3 fir_filter.py inputFF outputFF Included in the attached zip folder (fir.zip): fir_filter.py - Main filtering/downsampling script PFO_div5_fir.txt, PFO_div4_fir.txt, and PFO_div2_fir.txt - The coefficients used in the FIR filters to reduce the data rate by factors of 5, 4, and 2, respectively. ffPy - Folder containing the Python Flat File loading library ffCreator.py - Module used in fir_filter.py to generate flat files Build Environment: python >=3.6 numpy >=1.16 scipy >=1.3 PDS Environment: python3 = python 3.9 to use older version, use python3.8 or python3.6 instead of python3 Running the script: To run the script with the default parameters (reducing the data rate by a factor of 2 and then again by a factor of 5), one can run: python3 fir_filter.py inputFF outputFF where inputFF is the name of the flat file to be filtered/downsampled and outputFF is the name to give to the new file. Using a different ordering/set of factors: To run the script with a different set of factors, one should include the '--factorList' argument at the end, followed by an ordered list of factors (space-delimited). For example, to reduce a flat file's rate by a factor of 4 and then by a factor of 2, one should run: python3 fir_filter.py inputFF outputFF --factorList 4 2 Similarly, to reduce the rate by a single factor of 5, one can run: python3 fir_filter.py inputFF outputFF --factorList 5 At least one factor must passed when using the --factorList argument, but there is no limit to the total number of factors that can be passed. Additionally, each factor can be passed more than once in the list. Currently, the only accepted factors are 5, 4, and 2. # Insight FIR Filter + Downsampling script import argparse import numpy as np import sys sys.path.insert(0, 'ffPy') import FF_File from ffCreator import createFF class FIR_Filter(): def __init__(self, coeffs, factor): self.coeffs = coeffs self.revCoeffs = coeffs[::-1] self.factor = factor self.numCoef = len(coeffs) self.offset = int((self.numCoef + 1) / 2) def getTimeDelay(self, rate): return rate * (self.numCoef + 1) / 2 def applyCoeff(self, dtaSlice): return np.dot(self.revCoeffs, dtaSlice) def fillData(self, times, data, segments, filterBool=False): if len(segments) <= 1: return data, segments, times # Get indices representing a time gap rate = np.round(times[1]-times[0], 3) gapIndices = [] for z in range(1, len(segments)): a, b = segments[z-1] c, d = segments[z] gapIndices.append((b, c)) # Get the time ticks within the time gap and corresponding filler values timeFills = [self.getTimeFillVals(times,a,b,rate)[1:] for a,b in gapIndices] if not filterBool: valFills = [np.zeros(len(tVals)) for tVals in timeFills] else: valFills = [np.interp(tVals, times, data) for tVals in timeFills] # Insert filler values/times into arrays and adjust the indices # for each segment accordingly refIndex = 0 refIndices = [] for tVals, fillVals, (a,b) in zip(timeFills, valFills, gapIndices): data = np.insert(data, a+refIndex+1, fillVals) times = np.insert(times, a+refIndex+1, tVals) refIndex += len(tVals) refIndices.append(refIndex) # Adjust indices for segments to include the minimum number of points # necessary to calculate a value for its last point in the unfiltered data a, b = segments[0] newSegments = [(a, b+self.offset-1)] for (a, b), refIndex in zip(segments[1:], refIndices): newA = max(a+refIndex-self.offset, 0) newB = min(b+refIndex+self.offset-1, len(data) - 1) newSegments.append((newA, newB)) segments = newSegments return data, segments, times def filterDta(self, times, data, segments): data, segments, newTimes = self.fillData(times, data, segments, True) # Filter each data segment separated by time gaps dtaLst = [] for a, b in segments: segData = data[a:b+1] filtSeg = list(self.filterSegment(segData)) dtaLst.append(filtSeg) dta = np.concatenate(dtaLst) return dta, newTimes, segments def filterSegment(self, data): # Apply coefficients to data for i in range(self.numCoef, len(data), self.factor): yield self.applyCoeff(data[i-self.numCoef:i]) def downsample(self, times, data, segments): data, segments, newTimes = self.fillData(times, data, segments) # Downsample unfiltered data in accordance with the downsampled + filtered data dataLst = [] for a, b in segments: segData = data[a:b+1] segData = segData[self.offset:-self.offset+1:self.factor] dataLst.append(segData) ogDta = data data = np.concatenate(dataLst) return data def getTimeFillVals(self, times, sI, eI, rate): return np.arange(times[sI], times[eI], rate) def getNewTimes(self, times, segments): # Adjust times for each segment of data and then concatenate them timeLst = [] for a, b in segments: segTimes = times[a:b+1] filtTimes = self.getNewTimeSegment(segTimes) timeLst.append(filtTimes) times = np.concatenate(timeLst) return times def getNewTimeSegment(self, times): if len(times) <= self.numCoef: return [] # Calculate time delay and new time rate newTimes = times[self.offset:-self.offset+1:self.factor] return newTimes def ffDataDict(ffName): # Open flat file FID = FF_File.FF_ID(ffName, status=FF_File.FF_STATUS.READ | FF_File.FF_STATUS.EXIST) FID.open() # Extract data and column names nRows = FID.getRows() records = FID.DID.sliceArray(row=1, nRow=nRows) ffTime = records["time"] ffData = records["data"] colNames = FID.getColumnDescriptor("NAME")[1:] # Create dictionary ffDict = {} ffDict['Time'] = np.array(ffTime) for i, colName in enumerate(colNames): dta = np.array(ffData[:,i]) ffDict[colName] = dta units = FID.getColumnDescriptor('UNITS')[1:] if len(units) == 0: units = [''] * len(colNames) sources = FID.getColumnDescriptor('SOURCE')[1:] if len(sources) == 0: sources = [''] * len(colNames) # Close flat file and return result FID.close() epoch = 'Y1966' return ffDict, units, sources, colNames, epoch def setupParser(): parser = argparse.ArgumentParser(description='Downsample and filter Insight data') parser.add_argument('filename', metavar='FFName', help='Flat file to be corrected') parser.add_argument('output', metavar='FFOutput', help='Name of output flat file') parser.add_argument('--factorList', metavar='', type=int, nargs='+', help='Optional list of factors to downsample data by (overrides default)', default=None) args = parser.parse_args() return args.filename, args.output, args.factorList def readCoeffs(filename): f = open(filename, 'r') lines = f.readlines() coeffs = [float(line.strip(' ').strip('\n')) for line in lines] return coeffs def getDataSegments(times, res): # Split data into segments by time gaps diff = np.diff(times) indices = np.arange(0, len(diff), step=1) mask = diff > (res * 2) pts = indices[mask] # Points where time difference > res * 2 segments = [] lastIndex = 0 for index in pts: segments.append((lastIndex, index)) lastIndex = index + 1 segments.append((lastIndex, len(times)-1)) # Need to add last segment return segments def main(): # Create filter/downsampling objects div2 = 'PFO_div2_fir.txt' div4 = 'PFO_div4_fir.txt' div5 = 'PFO_div5_fir.txt' fir2 = FIR_Filter(readCoeffs(div2), 2) fir4 = FIR_Filter(readCoeffs(div4), 4) fir5 = FIR_Filter(readCoeffs(div5), 5) filtDict = {2: fir2, 4: fir4, 5: fir5} # Extract file info/data and filter order parameters inputFile, outputFile, filterOrder = setupParser() ffDict, units, sources, labels, epoch = ffDataDict(inputFile) fieldKws = ['Bx_IFG','By_IFG','Bz_IFG', 'Bx_SC', 'By_SC', 'Bz_SC'] # Successively apply filters and downsample data, replacing values in the # dictionary of flat file data/times factorList = [2, 5] if filterOrder is None else filterOrder for filt in [filtDict[factor] for factor in factorList]: # Get data segments (separated by time gaps) times = ffDict['Time'] res = np.round(times[1] - times[0], 3) segments = getDataSegments(times, res) # Times/segments should be updated after filtering data newTimes = times[:] newSegments = segments[:] # Apply changes to each data component individually for lbl in labels: if lbl in fieldKws: # Filter + downsample Bx, By, Bz newDta, newTimes, newSegments = filt.filterDta(times, ffDict[lbl], segments) ffDict[lbl] = newDta else: # Downsample everything else ffDict[lbl] = filt.downsample(times, ffDict[lbl], segments) # Special case for frequency component if lbl == 'freq': ffDict[lbl] /= filt.factor # Resample times ffDict['Time'] = filt.getNewTimes(newTimes, newSegments) # Generate the new flat file ffTime = ffDict['Time'] ffDta = np.array([ffDict[lbl] for lbl in labels]) ffDta = ffDta.T ffDict = {} # Free up memory hopefully createFF(outputFile, ffTime, ffDta, labels, units, sources, epoch) main() ============================================================================== PFO_div2_fir.txt ============================================================================== +5.40188E-07 +2.32249E-06 +3.63277E-06 -1.93396E-07 -8.95501E-06 -9.97237E-06 +8.01375E-06 +2.67262E-05 +8.06316E-06 -4.15263E-05 -4.60010E-05 +3.66205E-05 +1.02210E-04 +1.08291E-05 -1.55148E-04 -1.17739E-04 +1.63094E-04 +2.79102E-04 -7.22750E-05 -4.52733E-04 -1.62405E-04 +5.53390E-04 +5.47050E-04 -4.64885E-04 -1.01817E-03 +7.46139E-05 +1.42595E-03 +6.73124E-04 -1.54602E-03 -1.71851E-03 +1.12897E-03 +2.84387E-03 +1.48364E-05 -3.67150E-03 -1.91178E-03 +3.71947E-03 +4.33297E-03 -2.51848E-03 -6.74907E-03 -2.27038E-04 +8.36267E-03 +4.46951E-03 -8.23111E-03 -9.68691E-03 +5.46742E-03 +1.48322E-02 +5.20684E-04 -1.83824E-02 -9.81327E-03 +1.84633E-02 +2.18464E-02 -1.29443E-02 -3.54155E-02 -7.82490E-04 +4.88328E-02 +2.73601E-02 -6.02167E-02 -8.23896E-02 +6.78522E-02 +3.10064E-01 +4.29459E-01 +3.10064E-01 +6.78522E-02 -8.23896E-02 -6.02167E-02 +2.73601E-02 +4.88328E-02 -7.82490E-04 -3.54155E-02 -1.29443E-02 +2.18464E-02 +1.84633E-02 -9.81327E-03 -1.83824E-02 +5.20684E-04 +1.48322E-02 +5.46742E-03 -9.68691E-03 -8.23111E-03 +4.46951E-03 +8.36267E-03 -2.27038E-04 -6.74907E-03 -2.51848E-03 +4.33297E-03 +3.71947E-03 -1.91178E-03 -3.67150E-03 +1.48364E-05 +2.84387E-03 +1.12897E-03 -1.71851E-03 -1.54602E-03 +6.73124E-04 +1.42595E-03 +7.46139E-05 -1.01817E-03 -4.64885E-04 +5.47050E-04 +5.53390E-04 -1.62405E-04 -4.52733E-04 -7.22750E-05 +2.79102E-04 +1.63094E-04 -1.17739E-04 -1.55148E-04 +1.08291E-05 +1.02210E-04 +3.66205E-05 -4.60010E-05 -4.15263E-05 +8.06316E-06 +2.67262E-05 +8.01375E-06 -9.97237E-06 -8.95501E-06 -1.93396E-07 +3.63277E-06 +2.32249E-06 +5.40188E-07 ============================================================================== PFO_div4_fir.txt ============================================================================== -2.62103E-08 -1.16007E-07 -3.13840E-07 -6.33959E-07 -1.01372E-06 -1.28066E-06 -1.17060E-06 -4.24594E-07 +1.04357E-06 +2.97052E-06 +4.66393E-06 +5.12984E-06 +3.45085E-06 -6.68374E-07 -6.41479E-06 -1.17454E-05 -1.38743E-05 -1.03822E-05 -6.21545E-07 +1.32094E-05 +2.60325E-05 +3.13244E-05 +2.38375E-05 +2.67389E-06 -2.67384E-05 -5.31602E-05 -6.31275E-05 -4.68117E-05 -3.93904E-06 +5.31650E-05 +1.01792E-04 +1.16780E-04 +8.19053E-05 +1.02459E-07 -1.02782E-04 -1.84277E-04 -2.01188E-04 -1.29873E-04 +1.77821E-05 +1.91426E-04 +3.16877E-04 +3.25458E-04 +1.87417E-04 -6.55354E-05 -3.41682E-04 -5.19245E-04 -4.96741E-04 -2.44267E-04 +1.68223E-04 +5.83533E-04 +8.12905E-04 +7.17074E-04 +2.79720E-04 -3.62334E-04 -9.54170E-04 -1.21862E-03 -9.79341E-04 -2.58930E-04 +6.97502E-04 +1.49686E-03 +1.75277E-03 +1.26261E-03 +1.29137E-04 -1.23778E-03 -2.25915E-03 -2.42315E-03 -1.52712E-03 +1.84298E-04 +2.06315E-03 +3.29138E-03 +3.22490E-03 +1.70881E-03 -7.83227E-04 -3.27318E-03 -4.64737E-03 -4.13740E-03 -1.71226E-03 +1.80676E-03 +4.99750E-03 +6.39135E-03 +5.12302E-03 +1.39803E-03 -3.45483E-03 -7.42428E-03 -8.61979E-03 -6.12837E-03 -5.52730E-04 +6.04935E-03 +1.08768E-02 +1.15215E-02 +7.08831E-03 -1.19475E-03 -1.02089E-02 -1.60360E-02 -1.55553E-02 -7.93242E-03 +4.65119E-03 +1.74441E-02 +2.47318E-02 +2.21092E-02 +8.59317E-03 -1.22901E-02 -3.30081E-02 -4.41282E-02 -3.74333E-02 -9.01448E-03 +3.87734E-02 +9.72854E-02 +1.53647E-01 +1.94336E-01 +2.09159E-01 +1.94336E-01 +1.53647E-01 +9.72854E-02 +3.87734E-02 -9.01448E-03 -3.74333E-02 -4.41282E-02 -3.30081E-02 -1.22901E-02 +8.59317E-03 +2.21092E-02 +2.47318E-02 +1.74441E-02 +4.65119E-03 -7.93242E-03 -1.55553E-02 -1.60360E-02 -1.02089E-02 -1.19475E-03 +7.08831E-03 +1.15215E-02 +1.08768E-02 +6.04935E-03 -5.52730E-04 -6.12837E-03 -8.61979E-03 -7.42428E-03 -3.45483E-03 +1.39803E-03 +5.12302E-03 +6.39135E-03 +4.99750E-03 +1.80676E-03 -1.71226E-03 -4.13740E-03 -4.64737E-03 -3.27318E-03 -7.83227E-04 +1.70881E-03 +3.22490E-03 +3.29138E-03 +2.06315E-03 +1.84298E-04 -1.52712E-03 -2.42315E-03 -2.25915E-03 -1.23778E-03 +1.29137E-04 +1.26261E-03 +1.75277E-03 +1.49686E-03 +6.97502E-04 -2.58930E-04 -9.79341E-04 -1.21862E-03 -9.54170E-04 -3.62334E-04 +2.79720E-04 +7.17074E-04 +8.12905E-04 +5.83533E-04 +1.68223E-04 -2.44267E-04 -4.96741E-04 -5.19245E-04 -3.41682E-04 -6.55354E-05 +1.87417E-04 +3.25458E-04 +3.16877E-04 +1.91426E-04 +1.77821E-05 -1.29873E-04 -2.01188E-04 -1.84277E-04 -1.02782E-04 +1.02459E-07 +8.19053E-05 +1.16780E-04 +1.01792E-04 +5.31650E-05 -3.93904E-06 -4.68117E-05 -6.31275E-05 -5.31602E-05 -2.67384E-05 +2.67389E-06 +2.38375E-05 +3.13244E-05 +2.60325E-05 +1.32094E-05 -6.21545E-07 -1.03822E-05 -1.38743E-05 -1.17454E-05 -6.41479E-06 -6.68374E-07 +3.45085E-06 +5.12984E-06 +4.66393E-06 +2.97052E-06 +1.04357E-06 -4.24594E-07 -1.17060E-06 -1.28066E-06 -1.01372E-06 -6.33959E-07 -3.13840E-07 -1.16007E-07 -2.62103E-08 ============================================================================== PFO_div5_fir.txt ============================================================================== +2.42691E-08 +8.21473E-08 +1.82556E-07 +3.10908E-07 +4.15584E-07 +4.01145E-07 +1.38694E-07 -5.00010E-07 -1.58526E-06 -3.06029E-06 -4.68311E-06 -6.00936E-06 -6.44456E-06 -5.37998E-06 -2.40051E-06 +2.48009E-06 +8.63037E-06 +1.47461E-05 +1.90029E-05 +1.94442E-05 +1.45614E-05 +3.93760E-06 -1.12360E-05 -2.79889E-05 -4.19131E-05 -4.80738E-05 -4.23685E-05 -2.30363E-05 +8.12332E-06 +4.52898E-05 +7.92786E-05 +9.94125E-05 +9.63460E-05 +6.52114E-05 +8.19356E-06 -6.44014E-05 -1.35278E-04 -1.83655E-04 -1.90630E-04 -1.45180E-04 -4.91439E-05 +8.05611E-05 +2.13736E-04 +3.13012E-04 +3.43335E-04 +2.82663E-04 +1.30932E-04 -8.57153E-05 -3.17598E-04 -5.01563E-04 -5.76832E-04 -5.03014E-04 -2.75355E-04 +6.77340E-05 +4.48398E-04 +7.65141E-04 +9.18099E-04 +8.37964E-04 +5.10732E-04 -9.56978E-06 -6.05788E-04 -1.12145E-03 -1.39947E-03 -1.32684E-03 -8.73285E-04 -1.11819E-04 +7.87214E-04 +1.59076E-03 +2.06038E-03 +2.01907E-03 +1.40969E-03 +3.27223E-04 -9.87776E-04 -2.19792E-03 -2.95148E-03 -2.97956E-03 -2.18236E-03 -6.78615E-04 +1.20033E-03 +2.97700E-03 +4.14359E-03 +4.30043E-03 +3.28095E-03 +1.22655E-03 -1.41583E-03 -3.98165E-03 -5.74793E-03 -6.12751E-03 -4.84851E-03 -2.06729E-03 +1.62392E-03 +5.30927E-03 +7.96400E-03 +8.72455E-03 +7.14622E-03 +3.37640E-03 -1.81367E-03 -7.16356E-03 -1.12077E-02 -1.26498E-02 -1.07374E-02 -5.53706E-03 +1.97446E-03 +1.00493E-02 +1.65290E-02 +1.93592E-02 +1.71475E-02 +9.62708E-03 -2.09686E-03 -1.56032E-02 -2.75313E-02 -3.42217E-02 -3.24965E-02 -2.04140E-02 +2.17346E-03 +3.33986E-02 +6.95638E-02 +1.05716E-01 +1.36510E-01 +1.57186E-01 +1.64467E-01 +1.57186E-01 +1.36510E-01 +1.05716E-01 +6.95638E-02 +3.33986E-02 +2.17346E-03 -2.04140E-02 -3.24965E-02 -3.42217E-02 -2.75313E-02 -1.56032E-02 -2.09686E-03 +9.62708E-03 +1.71475E-02 +1.93592E-02 +1.65290E-02 +1.00493E-02 +1.97446E-03 -5.53706E-03 -1.07374E-02 -1.26498E-02 -1.12077E-02 -7.16356E-03 -1.81367E-03 +3.37640E-03 +7.14622E-03 +8.72455E-03 +7.96400E-03 +5.30927E-03 +1.62392E-03 -2.06729E-03 -4.84851E-03 -6.12751E-03 -5.74793E-03 -3.98165E-03 -1.41583E-03 +1.22655E-03 +3.28095E-03 +4.30043E-03 +4.14359E-03 +2.97700E-03 +1.20033E-03 -6.78615E-04 -2.18236E-03 -2.97956E-03 -2.95148E-03 -2.19792E-03 -9.87776E-04 +3.27223E-04 +1.40969E-03 +2.01907E-03 +2.06038E-03 +1.59076E-03 +7.87214E-04 -1.11819E-04 -8.73285E-04 -1.32684E-03 -1.39947E-03 -1.12145E-03 -6.05788E-04 -9.56978E-06 +5.10732E-04 +8.37964E-04 +9.18099E-04 +7.65141E-04 +4.48398E-04 +6.77340E-05 -2.75355E-04 -5.03014E-04 -5.76832E-04 -5.01563E-04 -3.17598E-04 -8.57153E-05 +1.30932E-04 +2.82663E-04 +3.43335E-04 +3.13012E-04 +2.13736E-04 +8.05611E-05 -4.91439E-05 -1.45180E-04 -1.90630E-04 -1.83655E-04 -1.35278E-04 -6.44014E-05 +8.19356E-06 +6.52114E-05 +9.63460E-05 +9.94125E-05 +7.92786E-05 +4.52898E-05 +8.12332E-06 -2.30363E-05 -4.23685E-05 -4.80738E-05 -4.19131E-05 -2.79889E-05 -1.12360E-05 +3.93760E-06 +1.45614E-05 +1.94442E-05 +1.90029E-05 +1.47461E-05 +8.63037E-06 +2.48009E-06 -2.40051E-06 -5.37998E-06 -6.44456E-06 -6.00936E-06 -4.68311E-06 -3.06029E-06 -1.58526E-06 -5.00010E-07 +1.38694E-07 +4.01145E-07 +4.15584E-07 +3.10908E-07 +1.82556E-07 +8.21473E-08 +2.42691E-08