Sped up the creation and filling of datasets

.gitignore

@@ -1,2 +1,2 @@
-GUIConfig.py
-*.pyc
+GUIConfig.py
+*.pyc

DataVaultListWidget.py

@@ -1,55 +1,55 @@
-from PyQt4 import QtGui
-from twisted.internet.defer import inlineCallbacks
-import socket
-
-
-class DataVaultList(QtGui.QWidget):
-
-    def __init__(self, tracename, parent=None):
-        super(DataVaultList, self).__init__()
-        self.tracename = tracename
-        self.connect()
-
-    @inlineCallbacks
-    def connect(self):
-        from labrad.wrappers import connectAsync
-        self.cxn = yield connectAsync(name=socket.gethostname() + ' Data Vault Client')
-        self.grapher = yield self.cxn.grapher
-        self.dv = yield self.cxn.data_vault
-        self.initializeGUI()
-
-    def initializeGUI(self):
-        mainLayout = QtGui.QVBoxLayout()
-        self.dataListWidget = QtGui.QListWidget()
-        self.dataListWidget.doubleClicked.connect(self.onDoubleclick)
-        mainLayout.addWidget(self.dataListWidget)
-        self.setWindowTitle('Data Vault')
-        self.setLayout(mainLayout)
-        self.populate()
-        self.show()
-
-    @inlineCallbacks
-    def populate(self):
-        self.dataListWidget.clear()
-        ls = yield self.dv.dir()
-        self.dataListWidget.addItem('...')
-        self.dataListWidget.addItems(sorted(ls[0]))
-        if ls[1] is not None:
-            self.dataListWidget.addItems(sorted(ls[1]))
-
-    @inlineCallbacks
-    def onDoubleclick(self, item):
-        item = self.dataListWidget.currentItem().text()
-        if item == '...':
-            yield self.dv.cd(1)
-            self.populate()
-        else:
-            try:
-                yield self.dv.cd(str(item))
-                self.populate()
-            except:
-                path = yield self.dv.cd()
-                yield self.grapher.plot((path, str(item)), self.tracename, False)
-
-    def closeEvent(self, event):
-        self.cxn.disconnect()
+from PyQt5 import QtWidgets
+from twisted.internet.defer import inlineCallbacks
+import socket
+
+
+class DataVaultList(QtWidgets.QWidget):
+
+    def __init__(self, tracename, parent=None):
+        super(DataVaultList, self).__init__()
+        self.tracename = tracename
+        self.connect()
+
+    @inlineCallbacks
+    def connect(self):
+        from labrad.wrappers import connectAsync
+        self.cxn = yield connectAsync(name=socket.gethostname() + ' Data Vault Client')
+        self.grapher = yield self.cxn.grapher
+        self.dv = yield self.cxn.data_vault
+        self.initializeGUI()
+
+    def initializeGUI(self):
+        mainLayout = QtWidgets.QVBoxLayout()
+        self.dataListWidget = QtWidgets.QListWidget()
+        self.dataListWidget.doubleClicked.connect(self.onDoubleclick)
+        mainLayout.addWidget(self.dataListWidget)
+        self.setWindowTitle('Data Vault')
+        self.setLayout(mainLayout)
+        self.populate()
+        self.show()
+
+    @inlineCallbacks
+    def populate(self):
+        self.dataListWidget.clear()
+        ls = yield self.dv.dir()
+        self.dataListWidget.addItem('...')
+        self.dataListWidget.addItems(sorted(ls[0]))
+        if ls[1] is not None:
+            self.dataListWidget.addItems(sorted(ls[1]))
+
+    @inlineCallbacks
+    def onDoubleclick(self, item):
+        item = self.dataListWidget.currentItem().text()
+        if item == '...':
+            yield self.dv.cd(1)
+            self.populate()
+        else:
+            try:
+                yield self.dv.cd(str(item))
+                self.populate()
+            except:
+                path = yield self.dv.cd()
+                yield self.grapher.plot((path, str(item)), self.tracename, False)
+
+    def closeEvent(self, event):
+        self.cxn.disconnect()

Dataset.py

@@ -1,82 +1,105 @@
-'''
-Parent class for datasets
-'''
-from twisted.internet.defer import inlineCallbacks, returnValue, DeferredLock, Deferred
-from PyQt4 import QtCore
-from twisted.internet.threads import deferToThread
-import numpy as np
-
-class Dataset(QtCore.QObject):
-
-    def __init__(self, data_vault, context, dataset_location,reactor):
-        super(Dataset, self).__init__()
-        self.data = None
-        self.accessingData = DeferredLock()
-        self.reactor = reactor
-        self.dataset_location = dataset_location
-        self.data_vault = data_vault
-        self.updateCounter = 0
-        self.context = context
-        self.connectDataVault()
-        self.setupListeners()
-
-    @inlineCallbacks
-    def connectDataVault(self):
-        yield self.data_vault.cd(self.dataset_location[0], context = self.context)
-        path, dataset_name = yield self.data_vault.open(self.dataset_location[1], context = self.context)
-        self.dataset_name = dataset_name
-
-    @inlineCallbacks
-    def setupListeners(self):
-        yield self.data_vault.signal__data_available(11111, context = self.context)
-        yield self.data_vault.addListener(listener = self.updateData, source = None, ID = 11111, context = self.context)
-
-
-    @inlineCallbacks
-    def openDataset(self):
-        yield self.data_vault.cd(self.dataset_location[0], context = self.context)
-        yield self.data_vault.open(self.dataset_location[1], context = self.context)
-
-    @inlineCallbacks
-    def getParameters(self):
-        parameters = yield self.data_vault.parameters(context = self.context)
-        parameterValues = []
-        for parameter in parameters:
-            parameterValue = yield self.data_vault.get_parameter(parameter, context = self.context)
-            parameterValues.append( (parameter, parameterValue) )
-        returnValue(parameterValues)
-
-    def updateData(self,x,y):
-        self.updateCounter += 1
-        self.getData()
-
-    @inlineCallbacks
-    def getData(self):
-        Data = yield self.data_vault.get(100, context = self.context)
-        if (self.data is None):
-            yield self.accessingData.acquire()
-            try:
-                self.data = Data.asarray
-            except:
-                self.data = Data
-            self.accessingData.release()
-        else:
-            yield self.accessingData.acquire()
-            try:
-                self.data = np.append(self.data, Data.asarray, 0)
-            except:
-                self.data = np.append(self.data, Data, 0)
-            self.accessingData.release()
-
-    @inlineCallbacks
-    def getLabels(self):
-        labels = []
-        yield self.openDataset()
-        variables = yield self.data_vault.variables(context = self.context)
-        for i in range(len(variables[1])):
-            labels.append(variables[1][i][1] + ' - ' + self.dataset_name)
-        returnValue(labels)
-
-    @inlineCallbacks
-    def disconnectDataSignal(self):
-        yield self.data_vault.removeListener(listener = self.updateData, source = None, ID = 11111, context = self.context)
+import numpy as np
+from PyQt5.QtCore import QObject
+from twisted.internet.defer import inlineCallbacks, returnValue, DeferredLock
+
+
+class Dataset(QObject):
+    '''
+    Parent class for datasets.
+    '''
+
+    def __init__(self, data_vault, context, dataset_location, reactor):
+        super(Dataset, self).__init__()
+        self.accessingData = DeferredLock()
+        self.reactor = reactor
+        self.context = context
+
+        # dataset storage variables
+        self.dataset_location = dataset_location
+        self.data_vault = data_vault
+        self.data = None
+        self.points_per_grab = 1000
+        self.last_index = 0
+        self.updateCounter = 0
+
+        # startup sequence
+        self.connectDataVault()
+        self.openDataset()
+        self.setupListeners()
+
+
+    # SETUP CONNECTION
+    @inlineCallbacks
+    def connectDataVault(self):
+        yield self.accessingData.acquire()
+        yield self.data_vault.cd(self.dataset_location[0], context=self.context)
+        _, self.dataset_name = yield self.data_vault.open(self.dataset_location[1], context=self.context)
+
+    @inlineCallbacks
+    def openDataset(self):
+        # open the dataset
+        yield self.data_vault.cd(self.dataset_location[0], context=self.context)
+        yield self.data_vault.open(self.dataset_location[1], context=self.context)
+        # allocate array size based on dataset size
+        dataset_shape = yield self.data_vault.shape(context=self.context)
+        self.data = np.zeros(dataset_shape)
+        self.accessingData.release()
+
+    @inlineCallbacks
+    def setupListeners(self):
+        yield self.data_vault.signal__data_available(11111, context=self.context)
+        yield self.data_vault.addListener(listener=self.updateData, source=None, ID=11111, context=self.context)
+
+    @inlineCallbacks
+    def disconnectDataSignal(self):
+        yield self.data_vault.removeListener(listener=self.updateData, source=None, ID=11111, context=self.context)
+
+
+    # GETTERS
+    @inlineCallbacks
+    def getParameters(self):
+        parameters = yield self.data_vault.parameters(context=self.context)
+        parameterValues = []
+        for parameter in parameters:
+            parameterValue = yield self.data_vault.get_parameter(parameter, context=self.context)
+            parameterValues.append((parameter, parameterValue))
+        returnValue(parameterValues)
+
+    @inlineCallbacks
+    def getLabels(self):
+        yield self.accessingData.acquire()
+        labels = []
+        _, all_dep = yield self.data_vault.variables(context=self.context)
+        for i in range(len(all_dep)):
+            label_tmp = all_dep[i][0] + ' - ' + self.dataset_name
+            if label_tmp in labels:
+                label_tmp += ' (' + str(i) + ')'
+            labels.append(label_tmp)
+        self.accessingData.release()
+        returnValue(labels)
+
+    def updateData(self, c, msg):
+        self.updateCounter += 1
+        self.getData()
+
+    @inlineCallbacks
+    def getData(self):
+        """
+        Gets data in bunches at a time and adds
+        them to self.data, which holds the dataset.
+        """
+        # acquire communication
+        yield self.accessingData.acquire()
+        # get data from the datavault
+        Data = yield self.data_vault.get(self.points_per_grab, context=self.context)
+        Data = np.array(Data)
+        next_index = self.last_index + np.shape(Data)[0]
+        # add to array if we have the space
+        if next_index <= np.shape(self.data)[0]:
+            self.data[self.last_index: next_index] = Data
+        # otherwise append to array
+        else:
+            self.data = np.append(self.data, Data, axis=0)
+        self.last_index = next_index
+        # release communication
+        self.accessingData.release()