From 9f701274200b08382205c8a7600b79f033812151 Mon Sep 17 00:00:00 2001 From: sprenger Date: Sat, 22 May 2021 23:11:16 +0200 Subject: [PATCH 01/18] [neuralynx] first draft for memory and speed improvements --- neo/rawio/neuralynxrawio/ncssections.py | 119 ++++++++++++++++-------- 1 file changed, 82 insertions(+), 37 deletions(-) diff --git a/neo/rawio/neuralynxrawio/ncssections.py b/neo/rawio/neuralynxrawio/ncssections.py index 135f04d74..cd376ac23 100644 --- a/neo/rawio/neuralynxrawio/ncssections.py +++ b/neo/rawio/neuralynxrawio/ncssections.py @@ -1,4 +1,6 @@ import math +import numpy as np +from memory_profiler import profile class NcsSections: @@ -127,21 +129,27 @@ def _parseGivenActualFrequency(ncsMemMap, ncsSects, chanNum, reqFreq, blkOnePred blkLen = 0 curBlock = ncsSects.sects[0] for recn in range(1, ncsMemMap.shape[0]): - if ncsMemMap['channel_id'][recn] != chanNum or \ - ncsMemMap['sample_rate'][recn] != reqFreq: + timestamp = ncsMemMap['timestamp'][recn] + channel_id = ncsMemMap['channel_id'][recn] + sample_rate = ncsMemMap['sample_rate'][recn] + nb_valid = ncsMemMap['nb_valid'][recn] + + # hdr = CscRecordHeader(ncsMemMap, recn) + if channel_id != chanNum or sample_rate != reqFreq: raise IOError('Channel number or sampling frequency changed in ' + 'records within file') predTime = NcsSectionsFactory.calc_sample_time(ncsSects.sampFreqUsed, startBlockPredTime, blkLen) - ts = ncsMemMap['timestamp'][recn] - nValidSamps = ncsMemMap['nb_valid'][recn] - if ts != predTime: + nValidSamps = nb_valid + if timestamp != predTime: curBlock.endRec = recn - 1 curBlock.endTime = predTime - curBlock = NcsSection(recn, ts, -1, -1) + curBlock = NcsSection(recn, timestamp, -1, -1) ncsSects.sects.append(curBlock) startBlockPredTime = NcsSectionsFactory.calc_sample_time( - ncsSects.sampFreqUsed, ts, nValidSamps) + ncsSects.sampFreqUsed, + timestamp, + nValidSamps) blkLen = 0 else: blkLen += nValidSamps @@ -213,6 +221,7 @@ def _buildGivenActualFrequency(ncsMemMap, actualSampFreq, reqFreq): blkOnePredTime) @staticmethod + # @profile def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): """ Parse blocks of records from file, allowing a maximum gap in timestamps between records @@ -233,6 +242,8 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): largest block """ + print('profiling for memory leak') + # track frequency of each block and use estimate with longest block maxBlkLen = 0 maxBlkFreqEstimate = 0 @@ -249,43 +260,77 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): curBlock = NcsSection(0, startBlockTime, -1, -1) ncsSects.sects.append(curBlock) - for recn in range(1, ncsMemMap.shape[0]): - if ncsMemMap['channel_id'][recn] != chanNum or \ - ncsMemMap['sample_rate'][recn] != recFreq: - raise IOError('Channel number or sampling frequency changed in ' + - 'records within file') - predTime = NcsSectionsFactory.calc_sample_time(ncsSects.sampFreqUsed, lastRecTime, - lastRecNumSamps) - ts = ncsMemMap['timestamp'][recn] - nb = ncsMemMap['nb_valid'][recn] - if abs(ts - predTime) > maxGapLen: - curBlock.endRec = recn - 1 - curBlock.endTime = predTime - curBlock = NcsSection(recn, ts, -1, -1) - ncsSects.sects.append(curBlock) - if blkLen > maxBlkLen: - maxBlkLen = blkLen - maxBlkFreqEstimate = (blkLen - lastRecNumSamps) * 1e6 / \ - (lastRecTime - startBlockTime) - startBlockTime = ts - blkLen = nb - else: - blkLen += nb - lastRecTime = ts - lastRecNumSamps = nb - if blkLen > maxBlkLen: - maxBlkFreqEstimate = (blkLen - lastRecNumSamps) * 1e6 / \ - (lastRecTime - startBlockTime) + # check for consistent channel_ids and sampling rates + if not all(ncsMemMap['channel_id'] == chanNum): + raise IOError('Channel number changed in records within file') + + if not all(ncsMemMap['sample_rate'] == recFreq): + raise IOError('Sampling frequency changed in records within file') + + + gap_rec_ids = list(np.where(ncsMemMap['nb_valid'] != ncsMemMap['nb_valid'][0])[0]) + + pred_times = ncsMemMap['timestamp'] + 1e6 / ncsSects.sampFreqUsed * ncsMemMap['nb_valid'][0] + max_pred_times = np.round(pred_times) + maxGapLen + delayed_recs = list(np.where(max_pred_times[:-1] <= ncsMemMap['timestamp'][1:])[0]) + + gap_rec_ids.extend(delayed_recs) + + for recn in gap_rec_ids: + curBlock = NcsSection(recn, ncsMemMap['timestamp'][recn], -1, -1) + ncsSects.sects.append(curBlock) + + # pred_times = round(startTime + 1e6 / ncsSects.sampFreqUsed * posn) + + + # for recn in range(1, ncsMemMap.shape[0]): + # timestamp = ncsMemMap['timestamp'][recn] + # channel_id = ncsMemMap['channel_id'][recn] + # sample_rate = ncsMemMap['sample_rate'][recn] + # nb_valid = ncsMemMap['nb_valid'][recn] + + # hdr = CscRecordHeader(ncsMemMap, recn) + # if channel_id != chanNum or sample_rate != recFreq: + # raise IOError('Channel number or sampling frequency changed in ' + + # # 'records within file') + # predTime = NcsSectionsFactory.calc_sample_time(ncsSects.sampFreqUsed, lastRecTime, + # lastRecNumSamps) + # + # + # if abs(timestamp - predTime) > maxGapLen: + # curBlock.endRec = recn - 1 + # curBlock.endTime = predTime + # curBlock = NcsSection(recn, timestamp, -1, -1) + # ncsSects.sects.append(curBlock) + # if blkLen > maxBlkLen: + # maxBlkLen = blkLen + # maxBlkFreqEstimate = (blkLen - lastRecNumSamps) * 1e6 / \ + # (lastRecTime - startBlockTime) + # startBlockTime = timestamp + # blkLen = nb_valid + # else: + # blkLen += nb_valid + # lastRecTime = timestamp + # lastRecNumSamps = nb_valid + # + # if blkLen > maxBlkLen: + # maxBlkFreqEstimate = (blkLen - lastRecNumSamps) * 1e6 / \ + # (lastRecTime - startBlockTime) curBlock.endRec = ncsMemMap.shape[0] - 1 endTime = NcsSectionsFactory.calc_sample_time(ncsSects.sampFreqUsed, lastRecTime, lastRecNumSamps) curBlock.endTime = endTime - ncsSects.sampFreqUsed = maxBlkFreqEstimate - ncsSects.microsPerSampUsed = NcsSectionsFactory.get_micros_per_samp_for_freq( - maxBlkFreqEstimate) + # TODO: This needs to be checked against original implementation + sample_freqs = ncsMemMap['nb_valid'][:-1] * 1e6 / np.diff(ncsMemMap['timestamp']) + mask = ~np.isin(np.arange(len(ncsMemMap['nb_valid'])), gap_rec_ids) + ncsSects.sampFreqUsed = np.mean(sample_freqs[mask[1:]]) + + # ncsSects.sampFreqUsed = maxBlkFreqEstimate + # ncsSects.microsPerSampUsed = NcsSectionsFactory.get_micros_per_samp_for_freq( + # maxBlkFreqEstimate) return ncsSects From 5d5029f52f4a319405eb267ad7c43faa41a4e770 Mon Sep 17 00:00:00 2001 From: sprenger Date: Thu, 10 Jun 2021 16:10:54 +0200 Subject: [PATCH 02/18] [neuralynx] temporary cleanup for travis tests --- neo/rawio/neuralynxrawio/ncssections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/neuralynxrawio/ncssections.py b/neo/rawio/neuralynxrawio/ncssections.py index cd376ac23..00835bda3 100644 --- a/neo/rawio/neuralynxrawio/ncssections.py +++ b/neo/rawio/neuralynxrawio/ncssections.py @@ -1,6 +1,6 @@ import math import numpy as np -from memory_profiler import profile +# from memory_profiler import profile class NcsSections: From d510545325ad00a4514e46c013d8722b5139a223 Mon Sep 17 00:00:00 2001 From: sprenger Date: Thu, 10 Jun 2021 17:39:23 +0200 Subject: [PATCH 03/18] [neuralynx] fix calculation of last record end time --- neo/rawio/neuralynxrawio/ncssections.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/neo/rawio/neuralynxrawio/ncssections.py b/neo/rawio/neuralynxrawio/ncssections.py index 00835bda3..84bb03f3d 100644 --- a/neo/rawio/neuralynxrawio/ncssections.py +++ b/neo/rawio/neuralynxrawio/ncssections.py @@ -268,7 +268,7 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): if not all(ncsMemMap['sample_rate'] == recFreq): raise IOError('Sampling frequency changed in records within file') - + # detect records with incomplete number of samples gap_rec_ids = list(np.where(ncsMemMap['nb_valid'] != ncsMemMap['nb_valid'][0])[0]) pred_times = ncsMemMap['timestamp'] + 1e6 / ncsSects.sampFreqUsed * ncsMemMap['nb_valid'][0] @@ -319,8 +319,9 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): # (lastRecTime - startBlockTime) curBlock.endRec = ncsMemMap.shape[0] - 1 - endTime = NcsSectionsFactory.calc_sample_time(ncsSects.sampFreqUsed, lastRecTime, - lastRecNumSamps) + endTime = NcsSectionsFactory.calc_sample_time(ncsSects.sampFreqUsed, ncsMemMap['timestamp'][-1], + ncsMemMap['nb_valid'][-1]) + curBlock.endTime = endTime # TODO: This needs to be checked against original implementation From 1141bde9d4230fc9256bb692a3b1fc30403f1cf9 Mon Sep 17 00:00:00 2001 From: sprenger Date: Fri, 11 Jun 2021 13:39:27 +0200 Subject: [PATCH 04/18] [neuralynx] fix extracted sections to be more similar to original implementation --- neo/rawio/neuralynxrawio/ncssections.py | 93 +++++++++++-------------- 1 file changed, 40 insertions(+), 53 deletions(-) diff --git a/neo/rawio/neuralynxrawio/ncssections.py b/neo/rawio/neuralynxrawio/ncssections.py index 84bb03f3d..a7f6bf43d 100644 --- a/neo/rawio/neuralynxrawio/ncssections.py +++ b/neo/rawio/neuralynxrawio/ncssections.py @@ -39,11 +39,12 @@ def __init__(self): self.endTime = -1 # end time of last record, that is, the end time of the last # sampling period contained in the last record of the section - def __init__(self, sb, st, eb, et): + def __init__(self, sb, st, eb, et, ns): self.startRec = sb self.startTime = st self.endRec = eb self.endTime = et + self.n_samples = ns def before_time(self, rhb): """ @@ -144,7 +145,7 @@ def _parseGivenActualFrequency(ncsMemMap, ncsSects, chanNum, reqFreq, blkOnePred if timestamp != predTime: curBlock.endRec = recn - 1 curBlock.endTime = predTime - curBlock = NcsSection(recn, timestamp, -1, -1) + curBlock = NcsSection(recn, timestamp, -1, -1, -1) ncsSects.sects.append(curBlock) startBlockPredTime = NcsSectionsFactory.calc_sample_time( ncsSects.sampFreqUsed, @@ -207,7 +208,7 @@ def _buildGivenActualFrequency(ncsMemMap, actualSampFreq, reqFreq): ncsMemMap['sample_rate'][lastBlkI] == reqFreq and \ lts == predLastBlockStartTime: lastBlkEndTime = NcsSectionsFactory.calc_sample_time(actualSampFreq, lts, lnb) - curBlock = NcsSection(0, ts0, lastBlkI, lastBlkEndTime) + curBlock = NcsSection(0, ts0, lastBlkI, lastBlkEndTime, -1) nb.sects.append(curBlock) return nb @@ -215,7 +216,10 @@ def _buildGivenActualFrequency(ncsMemMap, actualSampFreq, reqFreq): # otherwise need to scan looking for breaks else: blkOnePredTime = NcsSectionsFactory.calc_sample_time(actualSampFreq, ts0, nb0) - curBlock = NcsSection(0, ts0, -1, -1) + curBlock = NcsSection(0, ts0, -1, -1, -1) + blkOnePredTime = NcsSectionsFactory.calc_sample_time(actualSampFreq, rh0.timestamp, + rh0.nb_valid) + curBlock = NcsSection(0, rh0.timestamp, -1, -1, -1) nb.sects.append(curBlock) return NcsSectionsFactory._parseGivenActualFrequency(ncsMemMap, nb, chanNum, reqFreq, blkOnePredTime) @@ -244,6 +248,8 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): print('profiling for memory leak') + old_ncsSects = copy.deepcopy(ncsSects) + # track frequency of each block and use estimate with longest block maxBlkLen = 0 maxBlkFreqEstimate = 0 @@ -258,8 +264,8 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): lastRecNumSamps = blkLen recFreq = ncsMemMap['sample_rate'][0] - curBlock = NcsSection(0, startBlockTime, -1, -1) - ncsSects.sects.append(curBlock) + # curBlock = NcsSection(0, rh0.timestamp, -1, -1, -1) + # ncsSects.sects.append(curBlock) # check for consistent channel_ids and sampling rates if not all(ncsMemMap['channel_id'] == chanNum): @@ -277,57 +283,37 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): gap_rec_ids.extend(delayed_recs) - for recn in gap_rec_ids: - curBlock = NcsSection(recn, ncsMemMap['timestamp'][recn], -1, -1) - ncsSects.sects.append(curBlock) - - # pred_times = round(startTime + 1e6 / ncsSects.sampFreqUsed * posn) + # create recording segments from identified gaps + ncsSects.sects.append(NcsSection(0, ncsMemMap['timestamp'][0], -1, -1, -1)) + for gap_rec_id in gap_rec_ids: + curr_sec = ncsSects.sects[-1] + curr_sec.endRec = gap_rec_id + curr_sec.endTime = ncsMemMap['timestamp'][gap_rec_id] + n_samples = np.sum(ncsMemMap['nb_valid'][curr_sec.startRec:gap_rec_id+1]) + curr_sec.n_samples = n_samples + next_sec = NcsSection(gap_rec_id+1, ncsMemMap['timestamp'][gap_rec_id+1], -1, -1, -1) + ncsSects.sects.append(next_sec) - # for recn in range(1, ncsMemMap.shape[0]): - # timestamp = ncsMemMap['timestamp'][recn] - # channel_id = ncsMemMap['channel_id'][recn] - # sample_rate = ncsMemMap['sample_rate'][recn] - # nb_valid = ncsMemMap['nb_valid'][recn] + curr_sec = ncsSects.sects[-1] + curr_sec.endRec = len(ncsMemMap['timestamp']) -1 + curr_sec.endTime = ncsMemMap['timestamp'][-1] + n_samples = np.sum(ncsMemMap['nb_valid'][curr_sec.startRec:]) + curr_sec.n_samples = n_samples - # hdr = CscRecordHeader(ncsMemMap, recn) - # if channel_id != chanNum or sample_rate != recFreq: - # raise IOError('Channel number or sampling frequency changed in ' + - # # 'records within file') - # predTime = NcsSectionsFactory.calc_sample_time(ncsSects.sampFreqUsed, lastRecTime, - # lastRecNumSamps) - # - # - # if abs(timestamp - predTime) > maxGapLen: - # curBlock.endRec = recn - 1 - # curBlock.endTime = predTime - # curBlock = NcsSection(recn, timestamp, -1, -1) - # ncsSects.sects.append(curBlock) - # if blkLen > maxBlkLen: - # maxBlkLen = blkLen - # maxBlkFreqEstimate = (blkLen - lastRecNumSamps) * 1e6 / \ - # (lastRecTime - startBlockTime) - # startBlockTime = timestamp - # blkLen = nb_valid - # else: - # blkLen += nb_valid - # lastRecTime = timestamp - # lastRecNumSamps = nb_valid - # - # if blkLen > maxBlkLen: - # maxBlkFreqEstimate = (blkLen - lastRecNumSamps) * 1e6 / \ - # (lastRecTime - startBlockTime) - curBlock.endRec = ncsMemMap.shape[0] - 1 - endTime = NcsSectionsFactory.calc_sample_time(ncsSects.sampFreqUsed, ncsMemMap['timestamp'][-1], - ncsMemMap['nb_valid'][-1]) + # calculate the estimated frequency of the block with the most samples + max_blk_idx = np.argmax([bl.endRec - bl.startRec for bl in ncsSects.sects]) + max_blk = ncsSects.sects[max_blk_idx] - curBlock.endTime = endTime + maxBlkFreqEstimate = (max_blk.n_samples - ncsMemMap['nb_valid'][max_blk.endRec]) * 1e6 / \ + (max_blk.endTime - max_blk.startTime) # TODO: This needs to be checked against original implementation - sample_freqs = ncsMemMap['nb_valid'][:-1] * 1e6 / np.diff(ncsMemMap['timestamp']) - mask = ~np.isin(np.arange(len(ncsMemMap['nb_valid'])), gap_rec_ids) - ncsSects.sampFreqUsed = np.mean(sample_freqs[mask[1:]]) + ncsSects.sampFreqUsed = maxBlkFreqEstimate + ncsSects.microsPerSampUsed = NcsSectionsFactory.get_micros_per_samp_for_freq( + maxBlkFreqEstimate) + # ncsSects.sampFreqUsed = maxBlkFreqEstimate # ncsSects.microsPerSampUsed = NcsSectionsFactory.get_micros_per_samp_for_freq( @@ -369,9 +355,10 @@ def _buildForMaxGap(ncsMemMap, nomFreq): numSampsForPred = NcsSection._RECORD_SIZE * lastBlkI predLastBlockStartTime = NcsSectionsFactory.calc_sample_time(nomFreq, ts0, numSampsForPred) freqInFile = math.floor(nomFreq) - if lts - predLastBlockStartTime == 0 and lcid == chanNum and lsr == freqInFile: - endTime = NcsSectionsFactory.calc_sample_time(nomFreq, lts, lnb) - curBlock = NcsSection(0, ts0, lastBlkI, endTime) + if rhl.timestamp - predLastBlockStartTime == 0 and \ + rhl.channel_id == chanNum and rhl.sample_rate == freqInFile: + endTime = NcsSectionsFactory.calc_sample_time(nomFreq, rhl.timestamp, rhl.nb_valid) + curBlock = NcsSection(0, rh0.timestamp, lastBlkI, endTime, -1) nb.sects.append(curBlock) nb.sampFreqUsed = numSampsForPred / (lts - ts0) * 1e6 nb.microsPerSampUsed = NcsSectionsFactory.get_micros_per_samp_for_freq(nb.sampFreqUsed) From 76cf366523ca95af80379cb735c684df65b5f934 Mon Sep 17 00:00:00 2001 From: sprenger Date: Fri, 11 Jun 2021 13:44:50 +0200 Subject: [PATCH 05/18] [neuralynx] temporarily add copy module --- neo/rawio/neuralynxrawio/ncssections.py | 1 + 1 file changed, 1 insertion(+) diff --git a/neo/rawio/neuralynxrawio/ncssections.py b/neo/rawio/neuralynxrawio/ncssections.py index a7f6bf43d..4cca9321f 100644 --- a/neo/rawio/neuralynxrawio/ncssections.py +++ b/neo/rawio/neuralynxrawio/ncssections.py @@ -1,5 +1,6 @@ import math import numpy as np +import copy # from memory_profiler import profile From bbcab798f7d3e69ca706e23c5fec4bc8abb69084 Mon Sep 17 00:00:00 2001 From: sprenger Date: Fri, 11 Jun 2021 15:31:36 +0200 Subject: [PATCH 06/18] [neuralynx] first working version --- neo/rawio/neuralynxrawio/ncssections.py | 26 +++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/neo/rawio/neuralynxrawio/ncssections.py b/neo/rawio/neuralynxrawio/ncssections.py index 4cca9321f..751b9c468 100644 --- a/neo/rawio/neuralynxrawio/ncssections.py +++ b/neo/rawio/neuralynxrawio/ncssections.py @@ -257,6 +257,7 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): # Parse the record sequence, finding blocks of continuous time with no more than # maxGapLength and same channel number + chanNum = ncsMemMap['channel_id'][0] startBlockTime = ncsMemMap['timestamp'][0] @@ -265,6 +266,11 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): lastRecNumSamps = blkLen recFreq = ncsMemMap['sample_rate'][0] + rh0 = CscRecordHeader(ncsMemMap, 0) + chanNum = rh0.channel_id + recFreq = rh0.sample_rate + + # curBlock = NcsSection(0, rh0.timestamp, -1, -1, -1) # ncsSects.sects.append(curBlock) @@ -278,18 +284,26 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): # detect records with incomplete number of samples gap_rec_ids = list(np.where(ncsMemMap['nb_valid'] != ncsMemMap['nb_valid'][0])[0]) - pred_times = ncsMemMap['timestamp'] + 1e6 / ncsSects.sampFreqUsed * ncsMemMap['nb_valid'][0] - max_pred_times = np.round(pred_times) + maxGapLen + pred_times = np.rint(ncsMemMap['timestamp'] + 1e6 / ncsSects.sampFreqUsed * ncsMemMap['nb_valid']).astype(int) + max_pred_times = pred_times + maxGapLen delayed_recs = list(np.where(max_pred_times[:-1] <= ncsMemMap['timestamp'][1:])[0]) - gap_rec_ids.extend(delayed_recs) + # cleaning extracted gap ids + last_rec_id = len(ncsMemMap['timestamp'])-1 + if last_rec_id in gap_rec_ids: + gap_rec_ids.remove(last_rec_id) + gap_rec_ids = sorted(set(gap_rec_ids)) + + if len(ncsMemMap['timestamp'])-1 in gap_rec_ids: + print('here') + # create recording segments from identified gaps ncsSects.sects.append(NcsSection(0, ncsMemMap['timestamp'][0], -1, -1, -1)) for gap_rec_id in gap_rec_ids: curr_sec = ncsSects.sects[-1] curr_sec.endRec = gap_rec_id - curr_sec.endTime = ncsMemMap['timestamp'][gap_rec_id] + curr_sec.endTime = pred_times[gap_rec_id] n_samples = np.sum(ncsMemMap['nb_valid'][curr_sec.startRec:gap_rec_id+1]) curr_sec.n_samples = n_samples @@ -298,7 +312,7 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): curr_sec = ncsSects.sects[-1] curr_sec.endRec = len(ncsMemMap['timestamp']) -1 - curr_sec.endTime = ncsMemMap['timestamp'][-1] + curr_sec.endTime = pred_times[-1] n_samples = np.sum(ncsMemMap['nb_valid'][curr_sec.startRec:]) curr_sec.n_samples = n_samples @@ -308,7 +322,7 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): max_blk = ncsSects.sects[max_blk_idx] maxBlkFreqEstimate = (max_blk.n_samples - ncsMemMap['nb_valid'][max_blk.endRec]) * 1e6 / \ - (max_blk.endTime - max_blk.startTime) + (ncsMemMap['timestamp'][max_blk.endRec] - max_blk.startTime) # TODO: This needs to be checked against original implementation ncsSects.sampFreqUsed = maxBlkFreqEstimate From a27e08fa2781f3b87e154d8b7f66c0e4260fabf5 Mon Sep 17 00:00:00 2001 From: sprenger Date: Fri, 18 Jun 2021 15:56:01 +0200 Subject: [PATCH 07/18] remove outdated code --- neo/rawio/neuralynxrawio/ncssections.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/neo/rawio/neuralynxrawio/ncssections.py b/neo/rawio/neuralynxrawio/ncssections.py index 751b9c468..27311be4b 100644 --- a/neo/rawio/neuralynxrawio/ncssections.py +++ b/neo/rawio/neuralynxrawio/ncssections.py @@ -266,11 +266,6 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): lastRecNumSamps = blkLen recFreq = ncsMemMap['sample_rate'][0] - rh0 = CscRecordHeader(ncsMemMap, 0) - chanNum = rh0.channel_id - recFreq = rh0.sample_rate - - # curBlock = NcsSection(0, rh0.timestamp, -1, -1, -1) # ncsSects.sects.append(curBlock) From dd207f08b3b873270fdcc28999a3fefdd91e82d6 Mon Sep 17 00:00:00 2001 From: sprenger Date: Mon, 28 Jun 2021 18:42:02 +0200 Subject: [PATCH 08/18] [neuralynx] add memmap generating function --- neo/rawio/neuralynxrawio/neuralynxrawio.py | 54 ++++++++++++++++------ 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/neo/rawio/neuralynxrawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio/neuralynxrawio.py index 6b1ed9fd5..c62680524 100644 --- a/neo/rawio/neuralynxrawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio/neuralynxrawio.py @@ -21,6 +21,8 @@ import numpy as np import os +import pathlib +import copy from collections import (namedtuple, OrderedDict) from neo.rawio.neuralynxrawio.ncssections import (NcsSection, NcsSectionsFactory) @@ -182,14 +184,8 @@ def _parse_header(self): 'Several nse or ntt files have the same unit_id!!!' self.nse_ntt_filenames[chan_uid] = filename - dtype = get_nse_or_ntt_dtype(info, ext) - if os.path.getsize(filename) <= NlxHeader.HEADER_SIZE: - self._empty_nse_ntt.append(filename) - data = np.zeros((0,), dtype=dtype) - else: - data = np.memmap(filename, dtype=dtype, mode='r', - offset=NlxHeader.HEADER_SIZE) + data = self._get_file_map(filename) self._spike_memmap[chan_uid] = data @@ -222,8 +218,7 @@ def _parse_header(self): data = np.zeros((0,), dtype=nev_dtype) internal_ids = [] else: - data = np.memmap(filename, dtype=nev_dtype, mode='r', - offset=NlxHeader.HEADER_SIZE) + data = self._get_file_map(filename) internal_ids = np.unique(data[['event_id', 'ttl_input']]).tolist() for internal_event_id in internal_ids: if internal_event_id not in self.internal_event_ids: @@ -351,6 +346,38 @@ def _parse_header(self): # ~ ev_ann['digital_marker'] = # ~ ev_ann['analog_marker'] = + + def _get_file_map(self, filename): + """ + Create memory maps when needed + see also https://github.com/numpy/numpy/issues/19340 + """ + filename = pathlib.Path(filename) + suffix = filename.suffix + + if suffix == '.ncs': + return np.memmap(filename, dtype=self._ncs_dtype, mode='r', + offset=NlxHeader.HEADER_SIZE) + + elif suffix in ['.nse', '.ntt']: + info = NlxHeader(filename) + dtype = get_nse_or_ntt_dtype(info, suffix) + + # return empty map if file does not contain data + if os.path.getsize(filename) <= NlxHeader.HEADER_SIZE: + self._empty_nse_ntt.append(filename) + return np.zeros((0,), dtype=dtype) + + return np.memmap(filename, dtype=dtype, mode='r', + offset=NlxHeader.HEADER_SIZE) + + elif suffix == '.nev': + return np.memmap(filename, dtype=nev_dtype, mode='r', + offset=NlxHeader.HEADER_SIZE) + + else: + raise ValueError(f'Unknown file suffix {suffix}') + # Accessors for segment times which are offset by appropriate global start time def _segment_t_start(self, block_index, seg_index): return self._seg_t_starts[seg_index] - self.global_t_start @@ -538,16 +565,15 @@ def scan_ncs_files(self, ncs_filenames): chanSectMap = dict() for chan_uid, ncs_filename in self.ncs_filenames.items(): - data = np.memmap(ncs_filename, dtype=self._ncs_dtype, mode='r', - offset=NlxHeader.HEADER_SIZE) + data = self._get_file_map(ncs_filename) nlxHeader = NlxHeader(ncs_filename) if not chanSectMap or (chanSectMap and not NcsSectionsFactory._verifySectionsStructure(data, lastNcsSections)): lastNcsSections = NcsSectionsFactory.build_for_ncs_file(data, nlxHeader) - - chanSectMap[chan_uid] = [lastNcsSections, nlxHeader, data] + chanSectMap[chan_uid] = [lastNcsSections, nlxHeader, ncs_filename] + del data # Construct an inverse dictionary from NcsSections to list of associated chan_uids revSectMap = dict() @@ -568,7 +594,7 @@ def scan_ncs_files(self, ncs_filenames): # create segment with subdata block/t_start/t_stop/length for each channel for i, fileEntry in enumerate(self.ncs_filenames.items()): chan_uid = fileEntry[0] - data = chanSectMap[chan_uid][2] + data = self._get_file_map(chanSectMap[chan_uid][2]) # create a memmap for each record section of the current file curSects = chanSectMap[chan_uid][0] From de3f970bd0504805643e94636e4f69b0cb68eae2 Mon Sep 17 00:00:00 2001 From: sprenger Date: Mon, 28 Jun 2021 19:20:02 +0200 Subject: [PATCH 09/18] [neuralynx] fix dtype generation for memmaps --- neo/rawio/neuralynxrawio/neuralynxrawio.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/neo/rawio/neuralynxrawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio/neuralynxrawio.py index c62680524..db639d463 100644 --- a/neo/rawio/neuralynxrawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio/neuralynxrawio.py @@ -184,9 +184,7 @@ def _parse_header(self): 'Several nse or ntt files have the same unit_id!!!' self.nse_ntt_filenames[chan_uid] = filename - data = self._get_file_map(filename) - self._spike_memmap[chan_uid] = data unit_ids = np.unique(data['unit_id']) @@ -353,13 +351,13 @@ def _get_file_map(self, filename): see also https://github.com/numpy/numpy/issues/19340 """ filename = pathlib.Path(filename) - suffix = filename.suffix + suffix = filename.suffix.lower()[1:] - if suffix == '.ncs': + if suffix == 'ncs': return np.memmap(filename, dtype=self._ncs_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) - elif suffix in ['.nse', '.ntt']: + elif suffix in ['nse', 'ntt']: info = NlxHeader(filename) dtype = get_nse_or_ntt_dtype(info, suffix) @@ -371,7 +369,7 @@ def _get_file_map(self, filename): return np.memmap(filename, dtype=dtype, mode='r', offset=NlxHeader.HEADER_SIZE) - elif suffix == '.nev': + elif suffix == 'nev': return np.memmap(filename, dtype=nev_dtype, mode='r', offset=NlxHeader.HEADER_SIZE) From 6d9c73f9e0df79d867491f0db6a0371217d06254 Mon Sep 17 00:00:00 2001 From: sprenger Date: Mon, 28 Jun 2021 19:21:06 +0200 Subject: [PATCH 10/18] [axona] fix rebase with master and cleanup --- neo/rawio/neuralynxrawio/ncssections.py | 47 ++++++------------------- 1 file changed, 10 insertions(+), 37 deletions(-) diff --git a/neo/rawio/neuralynxrawio/ncssections.py b/neo/rawio/neuralynxrawio/ncssections.py index 27311be4b..10577b5df 100644 --- a/neo/rawio/neuralynxrawio/ncssections.py +++ b/neo/rawio/neuralynxrawio/ncssections.py @@ -218,9 +218,6 @@ def _buildGivenActualFrequency(ncsMemMap, actualSampFreq, reqFreq): else: blkOnePredTime = NcsSectionsFactory.calc_sample_time(actualSampFreq, ts0, nb0) curBlock = NcsSection(0, ts0, -1, -1, -1) - blkOnePredTime = NcsSectionsFactory.calc_sample_time(actualSampFreq, rh0.timestamp, - rh0.nb_valid) - curBlock = NcsSection(0, rh0.timestamp, -1, -1, -1) nb.sects.append(curBlock) return NcsSectionsFactory._parseGivenActualFrequency(ncsMemMap, nb, chanNum, reqFreq, blkOnePredTime) @@ -247,30 +244,12 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): largest block """ - print('profiling for memory leak') - - old_ncsSects = copy.deepcopy(ncsSects) - - # track frequency of each block and use estimate with longest block - maxBlkLen = 0 - maxBlkFreqEstimate = 0 - - # Parse the record sequence, finding blocks of continuous time with no more than - # maxGapLength and same channel number - chanNum = ncsMemMap['channel_id'][0] - - startBlockTime = ncsMemMap['timestamp'][0] - blkLen = ncsMemMap['nb_valid'][0] - lastRecTime = startBlockTime - lastRecNumSamps = blkLen recFreq = ncsMemMap['sample_rate'][0] - # curBlock = NcsSection(0, rh0.timestamp, -1, -1, -1) - # ncsSects.sects.append(curBlock) - # check for consistent channel_ids and sampling rates - if not all(ncsMemMap['channel_id'] == chanNum): + ncsMemMap['channel_id'] + if not (ncsMemMap['channel_id'] == chanNum).all(): raise IOError('Channel number changed in records within file') if not all(ncsMemMap['sample_rate'] == recFreq): @@ -285,7 +264,7 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): gap_rec_ids.extend(delayed_recs) # cleaning extracted gap ids - last_rec_id = len(ncsMemMap['timestamp'])-1 + last_rec_id = len(ncsMemMap['timestamp']) - 1 if last_rec_id in gap_rec_ids: gap_rec_ids.remove(last_rec_id) gap_rec_ids = sorted(set(gap_rec_ids)) @@ -299,14 +278,14 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): curr_sec = ncsSects.sects[-1] curr_sec.endRec = gap_rec_id curr_sec.endTime = pred_times[gap_rec_id] - n_samples = np.sum(ncsMemMap['nb_valid'][curr_sec.startRec:gap_rec_id+1]) + n_samples = np.sum(ncsMemMap['nb_valid'][curr_sec.startRec:gap_rec_id + 1]) curr_sec.n_samples = n_samples - next_sec = NcsSection(gap_rec_id+1, ncsMemMap['timestamp'][gap_rec_id+1], -1, -1, -1) + next_sec = NcsSection(gap_rec_id + 1, ncsMemMap['timestamp'][gap_rec_id + 1], -1, -1, -1) ncsSects.sects.append(next_sec) curr_sec = ncsSects.sects[-1] - curr_sec.endRec = len(ncsMemMap['timestamp']) -1 + curr_sec.endRec = len(ncsMemMap['timestamp']) - 1 curr_sec.endTime = pred_times[-1] n_samples = np.sum(ncsMemMap['nb_valid'][curr_sec.startRec:]) curr_sec.n_samples = n_samples @@ -319,16 +298,11 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): maxBlkFreqEstimate = (max_blk.n_samples - ncsMemMap['nb_valid'][max_blk.endRec]) * 1e6 / \ (ncsMemMap['timestamp'][max_blk.endRec] - max_blk.startTime) - # TODO: This needs to be checked against original implementation ncsSects.sampFreqUsed = maxBlkFreqEstimate ncsSects.microsPerSampUsed = NcsSectionsFactory.get_micros_per_samp_for_freq( maxBlkFreqEstimate) - - # ncsSects.sampFreqUsed = maxBlkFreqEstimate - # ncsSects.microsPerSampUsed = NcsSectionsFactory.get_micros_per_samp_for_freq( - # maxBlkFreqEstimate) - + del ncsMemMap return ncsSects @staticmethod @@ -365,10 +339,9 @@ def _buildForMaxGap(ncsMemMap, nomFreq): numSampsForPred = NcsSection._RECORD_SIZE * lastBlkI predLastBlockStartTime = NcsSectionsFactory.calc_sample_time(nomFreq, ts0, numSampsForPred) freqInFile = math.floor(nomFreq) - if rhl.timestamp - predLastBlockStartTime == 0 and \ - rhl.channel_id == chanNum and rhl.sample_rate == freqInFile: - endTime = NcsSectionsFactory.calc_sample_time(nomFreq, rhl.timestamp, rhl.nb_valid) - curBlock = NcsSection(0, rh0.timestamp, lastBlkI, endTime, -1) + if lts - predLastBlockStartTime == 0 and lcid == chanNum and lsr == freqInFile: + endTime = NcsSectionsFactory.calc_sample_time(nomFreq, lts, lnb) + curBlock = NcsSection(0, ts0, lastBlkI, endTime, -1) nb.sects.append(curBlock) nb.sampFreqUsed = numSampsForPred / (lts - ts0) * 1e6 nb.microsPerSampUsed = NcsSectionsFactory.get_micros_per_samp_for_freq(nb.sampFreqUsed) From 29e080aa1981dfc5ab0c18f535fa8a240eeeee99 Mon Sep 17 00:00:00 2001 From: sprenger Date: Tue, 29 Jun 2021 12:24:30 +0200 Subject: [PATCH 11/18] [neuralynx] cleanup and documentation --- neo/rawio/neuralynxrawio/ncssections.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/neo/rawio/neuralynxrawio/ncssections.py b/neo/rawio/neuralynxrawio/ncssections.py index 10577b5df..5420fb8cb 100644 --- a/neo/rawio/neuralynxrawio/ncssections.py +++ b/neo/rawio/neuralynxrawio/ncssections.py @@ -255,22 +255,27 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): if not all(ncsMemMap['sample_rate'] == recFreq): raise IOError('Sampling frequency changed in records within file') + # find most frequent number of samples + exp_nb_valid = np.argmax(np.bincount(ncsMemMap['nb_valid'])) # detect records with incomplete number of samples - gap_rec_ids = list(np.where(ncsMemMap['nb_valid'] != ncsMemMap['nb_valid'][0])[0]) + gap_rec_ids = list(np.where(ncsMemMap['nb_valid'] != exp_nb_valid)[0]) pred_times = np.rint(ncsMemMap['timestamp'] + 1e6 / ncsSects.sampFreqUsed * ncsMemMap['nb_valid']).astype(int) max_pred_times = pred_times + maxGapLen - delayed_recs = list(np.where(max_pred_times[:-1] <= ncsMemMap['timestamp'][1:])[0]) + # data records that start later than the predicted time (including the + # maximal accepted gap length) are considered delayed a gap is + # registered + delayed_recs = list(np.where(max_pred_times[:-1] < ncsMemMap['timestamp'][1:])[0]) gap_rec_ids.extend(delayed_recs) # cleaning extracted gap ids + # last record can not be the beginning of a gap last_rec_id = len(ncsMemMap['timestamp']) - 1 if last_rec_id in gap_rec_ids: gap_rec_ids.remove(last_rec_id) - gap_rec_ids = sorted(set(gap_rec_ids)) - if len(ncsMemMap['timestamp'])-1 in gap_rec_ids: - print('here') + # gap ids can only be listed once + gap_rec_ids = sorted(set(gap_rec_ids)) # create recording segments from identified gaps ncsSects.sects.append(NcsSection(0, ncsMemMap['timestamp'][0], -1, -1, -1)) @@ -290,7 +295,6 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): n_samples = np.sum(ncsMemMap['nb_valid'][curr_sec.startRec:]) curr_sec.n_samples = n_samples - # calculate the estimated frequency of the block with the most samples max_blk_idx = np.argmax([bl.endRec - bl.startRec for bl in ncsSects.sects]) max_blk = ncsSects.sects[max_blk_idx] @@ -301,7 +305,8 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): ncsSects.sampFreqUsed = maxBlkFreqEstimate ncsSects.microsPerSampUsed = NcsSectionsFactory.get_micros_per_samp_for_freq( maxBlkFreqEstimate) - + # free memory that is unnecessarily occupied by the memmap + # (see https://github.com/numpy/numpy/issues/19340) del ncsMemMap return ncsSects From 0877165f78426e307620718573851e81bec9f1b3 Mon Sep 17 00:00:00 2001 From: sprenger Date: Mon, 5 Jul 2021 12:56:23 +0200 Subject: [PATCH 12/18] [neuralynx] introduce equality and hashes for ncs section objects --- neo/rawio/neuralynxrawio/ncssections.py | 23 ++++++++++++- neo/test/rawiotest/test_neuralynxrawio.py | 40 ++++++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/neo/rawio/neuralynxrawio/ncssections.py b/neo/rawio/neuralynxrawio/ncssections.py index 5420fb8cb..bf1c5e7db 100644 --- a/neo/rawio/neuralynxrawio/ncssections.py +++ b/neo/rawio/neuralynxrawio/ncssections.py @@ -10,7 +10,7 @@ class NcsSections: Methods of NcsSectionsFactory perform parsing of this information from an Ncs file and produce these where the sections are discontiguous in time and in temporal order. - TODO: This class will likely need __eq__, __ne__, and __hash__ to be useful in + TODO: This class will likely need __ne__ to be useful in more sophisticated segment construction algorithms. """ @@ -19,6 +19,16 @@ def __init__(self): self.sampFreqUsed = 0 # actual sampling frequency of samples self.microsPerSampUsed = 0 # microseconds per sample + def __eq__(self, other): + samp_eq = self.sampFreqUsed == other.sampFreqUsed + micros_eq = self.microsPerSampUsed == other.microsPerSampUsed + sects_eq = self.sects == other.sects + return (samp_eq and micros_eq and sects_eq) + + def __hash__(self): + return (f'{self.sampFreqUsed};{self.microsPerSampUsed};' + f'{[s.__hash__() for s in self.sects]}').__hash__() + class NcsSection: """ @@ -47,6 +57,17 @@ def __init__(self, sb, st, eb, et, ns): self.endTime = et self.n_samples = ns + def __eq__(self, other): + return (self.startRec == other.startRec + and self.startTime == other.startTime + and self.endRec == other.endRec + and self.endTime == other.endTime + and self.n_samples == other.n_samples) + + def __hash__(self): + s = f'{self.startRec};{self.startTime};{self.endRec};{self.endTime};{self.n_samples}' + return s.__hash__() + def before_time(self, rhb): """ Determine if this section is completely before another section in time. diff --git a/neo/test/rawiotest/test_neuralynxrawio.py b/neo/test/rawiotest/test_neuralynxrawio.py index e2895a25c..c18268ad8 100644 --- a/neo/test/rawiotest/test_neuralynxrawio.py +++ b/neo/test/rawiotest/test_neuralynxrawio.py @@ -4,7 +4,8 @@ from neo.rawio.neuralynxrawio.neuralynxrawio import NeuralynxRawIO from neo.rawio.neuralynxrawio.nlxheader import NlxHeader -from neo.rawio.neuralynxrawio.ncssections import (NcsSections, NcsSectionsFactory) +from neo.rawio.neuralynxrawio.ncssections import (NcsSection, NcsSections, + NcsSectionsFactory) from neo.test.rawiotest.common_rawio_test import BaseTestRawIO import logging @@ -278,5 +279,42 @@ def test_block_verify(self): self.assertTrue(NcsSectionsFactory._verifySectionsStructure(data1, nb1)) +class TestNcsSections(TestNeuralynxRawIO, unittest.TestCase): + """ + Test building NcsBlocks for files of different revisions. + """ + entities_to_test = [] + + def test_equality(self): + ns0 = NcsSections() + ns1 = NcsSections() + + ns0.microsPerSampUsed = 1 + ns1.microsPerSampUsed = 1 + ns0.sampFreqUsed = 300 + ns1.sampFreqUsed = 300 + + self.assertEqual(ns0, ns1) + + # add sections + ns0.sects = [NcsSection(0, 0, 100, 100, 10)] + ns1.sects = [NcsSection(0, 0, 100, 100, 10)] + + self.assertEqual(ns0, ns1) + + # check inequality for different attributes + # different number of sections + ns0.sects.append(NcsSection(0, 0, 100, 100, 10)) + self.assertNotEqual(ns0, ns1) + + # different section attributes + ns0.sects = [NcsSection(0, 0, 200, 200, 10)] + self.assertNotEqual(ns0, ns1) + + # different attributes + ns0.sampFreqUsed = 400 + self.assertNotEqual(ns0, ns1) + + if __name__ == "__main__": unittest.main() From ae86e0e17c71efd6be4e2e71899a5d2046070e5a Mon Sep 17 00:00:00 2001 From: sprenger Date: Mon, 5 Jul 2021 12:56:44 +0200 Subject: [PATCH 13/18] [neuralynx] improve error messages --- neo/rawio/neuralynxrawio/neuralynxrawio.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/neo/rawio/neuralynxrawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio/neuralynxrawio.py index db639d463..8a92933a8 100644 --- a/neo/rawio/neuralynxrawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio/neuralynxrawio.py @@ -114,6 +114,11 @@ def _parse_header(self): filenames = sorted(os.listdir(self.dirname)) dirname = self.dirname else: + if not os.path.isfile(self.filename): + raise ValueError(f'Provided Filename is not a file: ' + f'{self.filename}. If you want to provide a ' + f'directory use the `dirname` keyword') + dirname, fname = os.path.split(self.filename) filenames = [fname] @@ -581,8 +586,8 @@ def scan_ncs_files(self, ncs_filenames): # If there is only one NcsSections structure in the set of ncs files, there should only # be one entry. Otherwise this is presently unsupported. if len(revSectMap) > 1: - raise IOError('ncs files have {} different sections structures. Unsupported.'.format( - len(revSectMap))) + raise IOError(f'ncs files have {len(revSectMap)} different sections ' + f'structures. Unsupported configuration.') seg_time_limits = SegmentTimeLimits(nb_segment=len(lastNcsSections.sects), t_start=[], t_stop=[], length=[], From 025c273828ae31ebdccbd8e496f81eefded6445d Mon Sep 17 00:00:00 2001 From: sprenger Date: Wed, 1 Sep 2021 15:58:08 +0200 Subject: [PATCH 14/18] [debug] fix dtype to int64 to avoid windows using int32 by default --- neo/rawio/neuralynxrawio/ncssections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/neuralynxrawio/ncssections.py b/neo/rawio/neuralynxrawio/ncssections.py index bf1c5e7db..100be69f7 100644 --- a/neo/rawio/neuralynxrawio/ncssections.py +++ b/neo/rawio/neuralynxrawio/ncssections.py @@ -281,7 +281,7 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): # detect records with incomplete number of samples gap_rec_ids = list(np.where(ncsMemMap['nb_valid'] != exp_nb_valid)[0]) - pred_times = np.rint(ncsMemMap['timestamp'] + 1e6 / ncsSects.sampFreqUsed * ncsMemMap['nb_valid']).astype(int) + pred_times = np.rint(ncsMemMap['timestamp'] + 1e6 / ncsSects.sampFreqUsed * ncsMemMap['nb_valid']).astype(np.int64) max_pred_times = pred_times + maxGapLen # data records that start later than the predicted time (including the # maximal accepted gap length) are considered delayed a gap is From 60da2f5dc1e2df56b88c6c9efdba193c5e978404 Mon Sep 17 00:00:00 2001 From: sprenger Date: Thu, 2 Sep 2021 13:57:44 +0200 Subject: [PATCH 15/18] [nixiofr] update tests to use unicode labels --- neo/test/iotest/test_nixio_fr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/test/iotest/test_nixio_fr.py b/neo/test/iotest/test_nixio_fr.py index 9adc3073b..d57b18ae2 100644 --- a/neo/test/iotest/test_nixio_fr.py +++ b/neo/test/iotest/test_nixio_fr.py @@ -112,10 +112,10 @@ def test_annotations(self): sp = SpikeTrain([3, 4, 5]* s, t_stop=10.0) sp.annotations['railway'] = 'hello train' ev = Event(np.arange(0, 30, 10)*pq.Hz, - labels=np.array(['trig0', 'trig1', 'trig2'], dtype='S')) + labels=np.array(['trig0', 'trig1', 'trig2'], dtype='U')) ev.annotations['venue'] = 'hello event' ev2 = Event(np.arange(0, 30, 10) * pq.Hz, - labels=np.array(['trig0', 'trig1', 'trig2'], dtype='S')) + labels=np.array(['trig0', 'trig1', 'trig2'], dtype='U')) ev2.annotations['evven'] = 'hello ev' seg.spiketrains.append(sp) seg.events.append(ev) From 9a6216b7f0f0c21a0562c72c7548260bad604f96 Mon Sep 17 00:00:00 2001 From: sprenger Date: Tue, 7 Sep 2021 11:00:14 +0200 Subject: [PATCH 16/18] [neuralynx] cleanup and refactoring --- neo/rawio/neuralynxrawio/ncssections.py | 23 +++++++++++----------- neo/rawio/neuralynxrawio/neuralynxrawio.py | 4 +++- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/neo/rawio/neuralynxrawio/ncssections.py b/neo/rawio/neuralynxrawio/ncssections.py index 100be69f7..3fb05b177 100644 --- a/neo/rawio/neuralynxrawio/ncssections.py +++ b/neo/rawio/neuralynxrawio/ncssections.py @@ -1,7 +1,5 @@ import math import numpy as np -import copy -# from memory_profiler import profile class NcsSections: @@ -149,7 +147,7 @@ def _parseGivenActualFrequency(ncsMemMap, ncsSects, chanNum, reqFreq, blkOnePred NcsSections object with block locations marked """ startBlockPredTime = blkOnePredTime - blkLen = 0 + blk_len = 0 curBlock = ncsSects.sects[0] for recn in range(1, ncsMemMap.shape[0]): timestamp = ncsMemMap['timestamp'][recn] @@ -157,30 +155,30 @@ def _parseGivenActualFrequency(ncsMemMap, ncsSects, chanNum, reqFreq, blkOnePred sample_rate = ncsMemMap['sample_rate'][recn] nb_valid = ncsMemMap['nb_valid'][recn] - # hdr = CscRecordHeader(ncsMemMap, recn) if channel_id != chanNum or sample_rate != reqFreq: raise IOError('Channel number or sampling frequency changed in ' + 'records within file') predTime = NcsSectionsFactory.calc_sample_time(ncsSects.sampFreqUsed, - startBlockPredTime, blkLen) + startBlockPredTime, blk_len) nValidSamps = nb_valid if timestamp != predTime: curBlock.endRec = recn - 1 curBlock.endTime = predTime + curBlock.n_samples = blk_len curBlock = NcsSection(recn, timestamp, -1, -1, -1) ncsSects.sects.append(curBlock) startBlockPredTime = NcsSectionsFactory.calc_sample_time( ncsSects.sampFreqUsed, timestamp, nValidSamps) - blkLen = 0 + blk_len = 0 else: - blkLen += nValidSamps + blk_len += nValidSamps curBlock.endRec = ncsMemMap.shape[0] - 1 endTime = NcsSectionsFactory.calc_sample_time(ncsSects.sampFreqUsed, startBlockPredTime, - blkLen) + blk_len) curBlock.endTime = endTime return ncsSects @@ -230,7 +228,8 @@ def _buildGivenActualFrequency(ncsMemMap, actualSampFreq, reqFreq): ncsMemMap['sample_rate'][lastBlkI] == reqFreq and \ lts == predLastBlockStartTime: lastBlkEndTime = NcsSectionsFactory.calc_sample_time(actualSampFreq, lts, lnb) - curBlock = NcsSection(0, ts0, lastBlkI, lastBlkEndTime, -1) + n_samples = NcsSection._RECORD_SIZE * lastBlkI + curBlock = NcsSection(0, ts0, lastBlkI, lastBlkEndTime, n_samples) nb.sects.append(curBlock) return nb @@ -244,7 +243,6 @@ def _buildGivenActualFrequency(ncsMemMap, actualSampFreq, reqFreq): blkOnePredTime) @staticmethod - # @profile def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): """ Parse blocks of records from file, allowing a maximum gap in timestamps between records @@ -307,7 +305,8 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): n_samples = np.sum(ncsMemMap['nb_valid'][curr_sec.startRec:gap_rec_id + 1]) curr_sec.n_samples = n_samples - next_sec = NcsSection(gap_rec_id + 1, ncsMemMap['timestamp'][gap_rec_id + 1], -1, -1, -1) + next_sec = NcsSection(gap_rec_id + 1, + ncsMemMap['timestamp'][gap_rec_id + 1], -1, -1, -1) ncsSects.sects.append(next_sec) curr_sec = ncsSects.sects[-1] @@ -367,7 +366,7 @@ def _buildForMaxGap(ncsMemMap, nomFreq): freqInFile = math.floor(nomFreq) if lts - predLastBlockStartTime == 0 and lcid == chanNum and lsr == freqInFile: endTime = NcsSectionsFactory.calc_sample_time(nomFreq, lts, lnb) - curBlock = NcsSection(0, ts0, lastBlkI, endTime, -1) + curBlock = NcsSection(0, ts0, lastBlkI, endTime, numSampsForPred) nb.sects.append(curBlock) nb.sampFreqUsed = numSampsForPred / (lts - ts0) * 1e6 nb.microsPerSampUsed = NcsSectionsFactory.get_micros_per_samp_for_freq(nb.sampFreqUsed) diff --git a/neo/rawio/neuralynxrawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio/neuralynxrawio.py index 48498cd57..1ddd47de9 100644 --- a/neo/rawio/neuralynxrawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio/neuralynxrawio.py @@ -112,6 +112,9 @@ def _source_name(self): else: return self.dirname + # from memory_profiler import profile + # + # @profile() def _parse_header(self): stream_channels = [] @@ -376,7 +379,6 @@ def _parse_header(self): # ~ ev_ann['digital_marker'] = # ~ ev_ann['analog_marker'] = - def _get_file_map(self, filename): """ Create memory maps when needed From a62d647b577ef39845f2e609e3cc451574811ef1 Mon Sep 17 00:00:00 2001 From: sprenger Date: Tue, 7 Sep 2021 11:04:16 +0200 Subject: [PATCH 17/18] [neuralynx] refactoring --- neo/rawio/neuralynxrawio/ncssections.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neo/rawio/neuralynxrawio/ncssections.py b/neo/rawio/neuralynxrawio/ncssections.py index 3fb05b177..7487c81b3 100644 --- a/neo/rawio/neuralynxrawio/ncssections.py +++ b/neo/rawio/neuralynxrawio/ncssections.py @@ -279,7 +279,8 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): # detect records with incomplete number of samples gap_rec_ids = list(np.where(ncsMemMap['nb_valid'] != exp_nb_valid)[0]) - pred_times = np.rint(ncsMemMap['timestamp'] + 1e6 / ncsSects.sampFreqUsed * ncsMemMap['nb_valid']).astype(np.int64) + rec_duration = 1e6 / ncsSects.sampFreqUsed * ncsMemMap['nb_valid'] + pred_times = np.rint(ncsMemMap['timestamp'] + rec_duration).astype(np.int64) max_pred_times = pred_times + maxGapLen # data records that start later than the predicted time (including the # maximal accepted gap length) are considered delayed a gap is From d76ea1f3cd4fa72a47f8149e5d0cfb14caf90d1f Mon Sep 17 00:00:00 2001 From: sprenger Date: Thu, 9 Sep 2021 11:05:10 +0200 Subject: [PATCH 18/18] [neuralynx] minor typo --- neo/rawio/neuralynxrawio/ncssections.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/rawio/neuralynxrawio/ncssections.py b/neo/rawio/neuralynxrawio/ncssections.py index 7487c81b3..016f49c89 100644 --- a/neo/rawio/neuralynxrawio/ncssections.py +++ b/neo/rawio/neuralynxrawio/ncssections.py @@ -283,8 +283,8 @@ def _parseForMaxGap(ncsMemMap, ncsSects, maxGapLen): pred_times = np.rint(ncsMemMap['timestamp'] + rec_duration).astype(np.int64) max_pred_times = pred_times + maxGapLen # data records that start later than the predicted time (including the - # maximal accepted gap length) are considered delayed a gap is - # registered + # maximal accepted gap length) are considered delayed and a gap is + # registered. delayed_recs = list(np.where(max_pred_times[:-1] < ncsMemMap['timestamp'][1:])[0]) gap_rec_ids.extend(delayed_recs)