Merge pull request #27 from spacetelescope/small-database-fixes

Small database fixes
spacetelescope · Aug 13, 2017 · 323e825 · 323e825
2 parents 7e29147 + cc3e859
commit 323e825
Show file tree

Hide file tree

Showing 31 changed files with 69 additions and 65 deletions.
diff --git a/acsql/database/database_interface.py b/acsql/database/database_interface.py
@@ -52,16 +52,16 @@
 from sqlalchemy import create_engine
 from sqlalchemy import Date
 from sqlalchemy import DateTime
-from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import Index
 from sqlalchemy import Enum
-from sqlalchemy import Float
 from sqlalchemy import ForeignKey
 from sqlalchemy import ForeignKeyConstraint
 from sqlalchemy import Integer
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy import String
 from sqlalchemy import Time
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.types import Float
 
 from acsql.utils.utils import SETTINGS
 
@@ -98,7 +98,7 @@ def define_columns(data_dict, class_name):
         elif keyword[1] == 'String':
             data_dict[keyword[0].lower()] = Column(String(50))
         elif keyword[1] == 'Float':
-            data_dict[keyword[0].lower()] = Column(Float())
+            data_dict[keyword[0].lower()] = Column(Float(precision=32))
         elif keyword[1] == 'Decimal':
             data_dict[keyword[0].lower()] = Column(Float(precision='13,8'))
         elif keyword[1] == 'Date':
@@ -212,7 +212,7 @@ def __init__(self, data_dict):
 
     __tablename__ = 'master'
     rootname = Column(String(8), primary_key=True, index=True, nullable=False)
-    path = Column(String(43), unique=True, nullable=False)
+    path = Column(String(15), unique=True, nullable=False)
     first_ingest_date = Column(Date, nullable=False)
     last_ingest_date = Column(Date, nullable=False)
     detector = Column(Enum('WFC', 'HRC', 'SBC'), nullable=False)

diff --git a/acsql/database/make_tabledefs.py b/acsql/database/make_tabledefs.py
@@ -69,7 +69,8 @@ def make_tabledefs(detector):
             print('Making file {}'.format(filename))
             with open(filename, 'w') as f:
                 for col in hsel.table.itercols():
-                    if col.name in ['ROOTNAME', 'Filename', 'FILENAME', 'Ext']:
+                    column_name = col.name
+                    if column_name in ['ROOTNAME', 'Filename', 'FILENAME', 'Ext']:
                         continue
                     elif col.dtype in [np.dtype('S68'), np.dtype('S80')]:
                         ptype = 'String'
@@ -81,9 +82,13 @@ def make_tabledefs(detector):
                         ptype = 'Float'
                     else:
                         print('Could not find type match: {}:{}'.format(
-                            col.name, col.dtype))
+                            column_name, col.dtype))
 
-                    f.write('{}, {}\n'.format(col.name, ptype))
+                    # If the column has a hyphen, switch it to underscore
+                    if '-' in column_name:
+                        column_name = column_name.replace('-', '_')
+
+                    f.write('{}, {}\n'.format(column_name, ptype))
 
 
 if __name__ == '__main__':

diff --git a/acsql/database/queries.py b/acsql/database/queries.py
@@ -314,10 +314,9 @@ def filenames_in_date_range(begin_date, end_date):
         performing the query.
     """
 
-    date_obs = getattr(WFC_raw_0, 'date-obs')  # Python doesn't like hyphens
     query = session.query(WFC_raw_0.filename)\
-        .filter(date_obs >= begin_date)\
-        .filter(date_obs <= end_date)
+        .filter(WFC_raw_0.date_obs >= begin_date)\
+        .filter(WFC_raw_0.date_obs <= end_date)
     query_results = query.all()
 
     print('\nQuery performed:\n\n{}\n'.format(str(query)))

diff --git a/acsql/database/table_definitions/hrc_crj_0.txt b/acsql/database/table_definitions/hrc_crj_0.txt
@@ -44,7 +44,7 @@ BIASLEVD, Float
 PHOTCORR, String
 CCDOFSTB, Integer
 NAXIS, Integer
-DATE-OBS, String
+DATE_OBS, String
 FWSOFFST, Integer
 POSTARG1, Float
 OSCNTAB, String
@@ -75,7 +75,7 @@ ATODCORR, String
 BADINPDQ, Integer
 REJ_RATE, Float
 ATODGND, Float
-TIME-OBS, String
+TIME_OBS, String
 DIRIMAGE, String
 FLASHSTA, String
 BITPIX, Integer

diff --git a/acsql/database/table_definitions/hrc_drz_0.txt b/acsql/database/table_definitions/hrc_drz_0.txt
@@ -51,7 +51,7 @@ PYWCSVER, String
 PHOTCORR, String
 D002KERN, String
 NAXIS, Integer
-DATE-OBS, String
+DATE_OBS, String
 FWSOFFST, Integer
 POSTARG1, Float
 OSCNTAB, String
@@ -542,7 +542,7 @@ D027FVAL, String
 D032ISCL, String
 D042VER, String
 D037GEOM, String
-TIME-OBS, String
+TIME_OBS, String
 ORIGIN, String
 D017OUUN, String
 D046MASK, String

diff --git a/acsql/database/table_definitions/hrc_flt_0.txt b/acsql/database/table_definitions/hrc_flt_0.txt
@@ -44,7 +44,7 @@ BIASLEVD, Float
 PHOTCORR, String
 CCDOFSTB, Integer
 NAXIS, Integer
-DATE-OBS, String
+DATE_OBS, String
 FWSOFFST, Integer
 POSTARG1, Float
 OSCNTAB, String
@@ -75,7 +75,7 @@ ATODCORR, String
 BADINPDQ, Integer
 REJ_RATE, Float
 ATODGND, Float
-TIME-OBS, String
+TIME_OBS, String
 DIRIMAGE, String
 FLASHSTA, String
 BITPIX, Integer

diff --git a/acsql/database/table_definitions/hrc_jif_1.txt b/acsql/database/table_definitions/hrc_jif_1.txt
@@ -67,4 +67,4 @@ GS_GAP, Float
 GSFAIL, String
 SGSTAR, String
 NO_TLM, String
-TDF-DOWN, Bool
+TDF_DOWN, Bool
diff --git a/acsql/database/table_definitions/hrc_jif_2.txt b/acsql/database/table_definitions/hrc_jif_2.txt
@@ -63,7 +63,7 @@ N_TM_GAP, Integer
 SLEWING, String
 TLM_PROB, Bool
 TM_GAP, Float
-TDF-DOWN, Bool
+TDF_DOWN, Bool
 GS_GAP, String
 GSFAIL, String
 SGSTAR, String

diff --git a/acsql/database/table_definitions/hrc_raw_0.txt b/acsql/database/table_definitions/hrc_raw_0.txt
@@ -44,7 +44,7 @@ BIASLEVD, Float
 PHOTCORR, String
 CCDOFSTB, Integer
 NAXIS, Integer
-DATE-OBS, String
+DATE_OBS, String
 FWSOFFST, Integer
 POSTARG1, Float
 OSCNTAB, String
@@ -75,7 +75,7 @@ ATODCORR, String
 BADINPDQ, Integer
 REJ_RATE, Float
 ATODGND, Float
-TIME-OBS, String
+TIME_OBS, String
 DIRIMAGE, String
 FLASHSTA, String
 BITPIX, Integer

diff --git a/acsql/database/table_definitions/sbc_drz_0.txt b/acsql/database/table_definitions/sbc_drz_0.txt
@@ -40,7 +40,7 @@ PYWCSVER, String
 PHOTCORR, String
 D002KERN, String
 NAXIS, Integer
-DATE-OBS, String
+DATE_OBS, String
 FWSOFFST, Integer
 POSTARG1, Float
 OSCNTAB, String
@@ -951,7 +951,7 @@ PR_INV_F, String
 PROCTIME, Float
 LINENUM, String
 ASN_ID, String
-TIME-OBS, String
+TIME_OBS, String
 PR_INV_L, String
 ASN_TAB, String
 PA_V3, Float

diff --git a/acsql/database/table_definitions/sbc_flt_0.txt b/acsql/database/table_definitions/sbc_flt_0.txt
@@ -36,7 +36,7 @@ LFLTFILE, String
 PYWCSVER, String
 PHOTCORR, String
 NAXIS, Integer
-DATE-OBS, String
+DATE_OBS, String
 FWSOFFST, Integer
 POSTARG1, Float
 OSCNTAB, String
@@ -59,7 +59,7 @@ DRIZCORR, String
 RA_TARG, Float
 PA_V3, Float
 P1_SHAPE, String
-TIME-OBS, String
+TIME_OBS, String
 DIRIMAGE, String
 BITPIX, Integer
 P1_NPTS, Integer

diff --git a/acsql/database/table_definitions/sbc_jif_1.txt b/acsql/database/table_definitions/sbc_jif_1.txt
@@ -65,6 +65,6 @@ TM_GAP, String
 TLM_PROB, String
 SLEWING, String
 N_TM_GAP, String
-TDF-DOWN, String
+TDF_DOWN, String
 GSFAIL, String
 SGSTAR, String
diff --git a/acsql/database/table_definitions/sbc_jif_2.txt b/acsql/database/table_definitions/sbc_jif_2.txt
@@ -64,6 +64,6 @@ TLM_PROB, String
 N_TM_GAP, String
 GS_GAP, Float
 TM_GAP, String
-TDF-DOWN, String
+TDF_DOWN, String
 GSFAIL, String
 SGSTAR, String
diff --git a/acsql/database/table_definitions/sbc_raw_0.txt b/acsql/database/table_definitions/sbc_raw_0.txt
@@ -34,7 +34,7 @@ EXPTIME, Float
 LFLTFILE, String
 PHOTCORR, String
 NAXIS, Integer
-DATE-OBS, String
+DATE_OBS, String
 FWSOFFST, Integer
 POSTARG1, Float
 OSCNTAB, String
@@ -57,7 +57,7 @@ DRIZCORR, String
 RA_TARG, Float
 PA_V3, Float
 P1_SHAPE, String
-TIME-OBS, String
+TIME_OBS, String
 DIRIMAGE, String
 BITPIX, Integer
 P1_NPTS, Integer

diff --git a/acsql/database/table_definitions/wfc_crc_0.txt b/acsql/database/table_definitions/wfc_crc_0.txt
@@ -46,7 +46,7 @@ BIASLEVD, Float
 PHOTCORR, String
 CCDOFSTB, Integer
 NAXIS, Integer
-DATE-OBS, String
+DATE_OBS, String
 FWSOFFST, Integer
 POSTARG1, Float
 OSCNTAB, String
@@ -78,7 +78,7 @@ ATODCORR, String
 BADINPDQ, Integer
 REJ_RATE, Float
 ATODGND, Float
-TIME-OBS, String
+TIME_OBS, String
 DIRIMAGE, String
 CTE_VER, String
 FLASHSTA, String

diff --git a/acsql/database/table_definitions/wfc_crj_0.txt b/acsql/database/table_definitions/wfc_crj_0.txt
@@ -44,7 +44,7 @@ BIASLEVD, Float
 PHOTCORR, String
 CCDOFSTB, Integer
 NAXIS, Integer
-DATE-OBS, String
+DATE_OBS, String
 FWSOFFST, Integer
 POSTARG1, Float
 OSCNTAB, String
@@ -75,7 +75,7 @@ ATODCORR, String
 BADINPDQ, Integer
 REJ_RATE, Float
 ATODGND, Float
-TIME-OBS, String
+TIME_OBS, String
 DIRIMAGE, String
 FLASHSTA, String
 BITPIX, Integer

diff --git a/acsql/database/table_definitions/wfc_drc_0.txt b/acsql/database/table_definitions/wfc_drc_0.txt
@@ -81,7 +81,7 @@ D002KERN, String
 D008COEF, String
 D007OUCO, String
 NAXIS, Integer
-DATE-OBS, String
+DATE_OBS, String
 FWSOFFST, Integer
 POSTARG1, Float
 OSCNTAB, String

diff --git a/acsql/database/table_definitions/wfc_drz_0.txt b/acsql/database/table_definitions/wfc_drz_0.txt
@@ -80,7 +80,7 @@ D002KERN, String
 D008COEF, String
 D007OUCO, String
 NAXIS, Integer
-DATE-OBS, String
+DATE_OBS, String
 FWSOFFST, Integer
 POSTARG1, Float
 OSCNTAB, String
@@ -674,7 +674,7 @@ D034OUCO, String
 D040OUUN, String
 D033VER, String
 D033GEOM, String
-TIME-OBS, String
+TIME_OBS, String
 CTE_NAME, String
 D038PIXF, Float
 OPUS_VER, String

diff --git a/acsql/database/table_definitions/wfc_flc_0.txt b/acsql/database/table_definitions/wfc_flc_0.txt
@@ -46,7 +46,7 @@ BIASLEVD, Float
 PHOTCORR, String
 CCDOFSTB, Integer
 NAXIS, Integer
-DATE-OBS, String
+DATE_OBS, String
 FWSOFFST, Integer
 POSTARG1, Float
 OSCNTAB, String
@@ -78,7 +78,7 @@ ATODCORR, String
 BADINPDQ, Integer
 REJ_RATE, Float
 ATODGND, Float
-TIME-OBS, String
+TIME_OBS, String
 DIRIMAGE, String
 CTE_VER, String
 FLASHSTA, String

diff --git a/acsql/database/table_definitions/wfc_flt_0.txt b/acsql/database/table_definitions/wfc_flt_0.txt
@@ -44,7 +44,7 @@ BIASLEVD, Float
 PHOTCORR, String
 CCDOFSTB, Integer
 NAXIS, Integer
-DATE-OBS, String
+DATE_OBS, String
 FWSOFFST, Integer
 POSTARG1, Float
 OSCNTAB, String
@@ -75,7 +75,7 @@ ATODCORR, String
 BADINPDQ, Integer
 REJ_RATE, Float
 ATODGND, Float
-TIME-OBS, String
+TIME_OBS, String
 DIRIMAGE, String
 FLASHSTA, String
 BITPIX, Integer

diff --git a/acsql/database/table_definitions/wfc_jif_1.txt b/acsql/database/table_definitions/wfc_jif_1.txt
@@ -67,4 +67,4 @@ GSFAIL, String
 TM_GAP, String
 NO_TLM, String
 SLEWING, Bool
-TDF-DOWN, Bool
+TDF_DOWN, Bool
diff --git a/acsql/database/table_definitions/wfc_jif_2.txt b/acsql/database/table_definitions/wfc_jif_2.txt
@@ -65,6 +65,6 @@ TLM_PROB, String
 N_TM_GAP, String
 GS_GAP, Float
 SGSTAR, String
-TDF-DOWN, Bool
+TDF_DOWN, Bool
 NO_TLM, String
 SLEWING, Bool
diff --git a/acsql/database/table_definitions/wfc_jif_3.txt b/acsql/database/table_definitions/wfc_jif_3.txt
@@ -65,6 +65,6 @@ TM_GAP, Float
 GS_GAP, Float
 SGSTAR, String
 GSFAIL, String
-TDF-DOWN, Bool
+TDF_DOWN, Bool
 NO_TLM, Bool
 SLEWING, Bool
diff --git a/acsql/database/table_definitions/wfc_jif_4.txt b/acsql/database/table_definitions/wfc_jif_4.txt
@@ -65,6 +65,6 @@ N_TM_GAP, Integer
 GS_GAP, Float
 SGSTAR, String
 GSFAIL, String
-TDF-DOWN, Bool
+TDF_DOWN, Bool
 NO_TLM, Bool
 SLEWING, Bool
diff --git a/acsql/database/table_definitions/wfc_jif_5.txt b/acsql/database/table_definitions/wfc_jif_5.txt
@@ -63,7 +63,7 @@ APERTURE, String
 OBSERVTN, String
 GUIDEACT, String
 SGSTAR, String
-TDF-DOWN, Bool
+TDF_DOWN, Bool
 GS_GAP, Float
 NO_TLM, Bool
 GSFAIL, String

diff --git a/acsql/database/table_definitions/wfc_jif_6.txt b/acsql/database/table_definitions/wfc_jif_6.txt
@@ -46,7 +46,7 @@ TLMFORM, String
 APER_V2, Float
 TM_GAP, Float
 BITPIX, Integer
-TDF-DOWN, Bool
+TDF_DOWN, Bool
 XPIXINC, Float
 OBSERVTN, String
 LOS_MOON, Float

diff --git a/acsql/database/table_definitions/wfc_raw_0.txt b/acsql/database/table_definitions/wfc_raw_0.txt
@@ -44,7 +44,7 @@ BIASLEVD, Float
 PHOTCORR, String
 CCDOFSTB, Integer
 NAXIS, Integer
-DATE-OBS, String
+DATE_OBS, String
 FWSOFFST, Integer
 POSTARG1, Float
 OSCNTAB, String
@@ -75,7 +75,7 @@ ATODCORR, String
 BADINPDQ, Integer
 REJ_RATE, Float
 ATODGND, Float
-TIME-OBS, String
+TIME_OBS, String
 DIRIMAGE, String
 FLASHSTA, String
 BITPIX, Integer