diff --git a/acsql/database/database_interface.py b/acsql/database/database_interface.py index bfffb3a..5486e28 100644 --- a/acsql/database/database_interface.py +++ b/acsql/database/database_interface.py @@ -52,16 +52,16 @@ from sqlalchemy import create_engine from sqlalchemy import Date from sqlalchemy import DateTime -from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import Index from sqlalchemy import Enum -from sqlalchemy import Float from sqlalchemy import ForeignKey from sqlalchemy import ForeignKeyConstraint from sqlalchemy import Integer from sqlalchemy.orm import sessionmaker from sqlalchemy import String from sqlalchemy import Time +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.types import Float from acsql.utils.utils import SETTINGS @@ -98,7 +98,7 @@ def define_columns(data_dict, class_name): elif keyword[1] == 'String': data_dict[keyword[0].lower()] = Column(String(50)) elif keyword[1] == 'Float': - data_dict[keyword[0].lower()] = Column(Float()) + data_dict[keyword[0].lower()] = Column(Float(precision=32)) elif keyword[1] == 'Decimal': data_dict[keyword[0].lower()] = Column(Float(precision='13,8')) elif keyword[1] == 'Date': @@ -212,7 +212,7 @@ def __init__(self, data_dict): __tablename__ = 'master' rootname = Column(String(8), primary_key=True, index=True, nullable=False) - path = Column(String(43), unique=True, nullable=False) + path = Column(String(15), unique=True, nullable=False) first_ingest_date = Column(Date, nullable=False) last_ingest_date = Column(Date, nullable=False) detector = Column(Enum('WFC', 'HRC', 'SBC'), nullable=False) diff --git a/acsql/database/make_tabledefs.py b/acsql/database/make_tabledefs.py index e0a3309..ae7ca9f 100644 --- a/acsql/database/make_tabledefs.py +++ b/acsql/database/make_tabledefs.py @@ -69,7 +69,8 @@ def make_tabledefs(detector): print('Making file {}'.format(filename)) with open(filename, 'w') as f: for col in hsel.table.itercols(): - if col.name in ['ROOTNAME', 'Filename', 'FILENAME', 'Ext']: + column_name = col.name + if column_name in ['ROOTNAME', 'Filename', 'FILENAME', 'Ext']: continue elif col.dtype in [np.dtype('S68'), np.dtype('S80')]: ptype = 'String' @@ -81,9 +82,13 @@ def make_tabledefs(detector): ptype = 'Float' else: print('Could not find type match: {}:{}'.format( - col.name, col.dtype)) + column_name, col.dtype)) - f.write('{}, {}\n'.format(col.name, ptype)) + # If the column has a hyphen, switch it to underscore + if '-' in column_name: + column_name = column_name.replace('-', '_') + + f.write('{}, {}\n'.format(column_name, ptype)) if __name__ == '__main__': diff --git a/acsql/database/queries.py b/acsql/database/queries.py index fb8053e..7b1a908 100644 --- a/acsql/database/queries.py +++ b/acsql/database/queries.py @@ -314,10 +314,9 @@ def filenames_in_date_range(begin_date, end_date): performing the query. """ - date_obs = getattr(WFC_raw_0, 'date-obs') # Python doesn't like hyphens query = session.query(WFC_raw_0.filename)\ - .filter(date_obs >= begin_date)\ - .filter(date_obs <= end_date) + .filter(WFC_raw_0.date_obs >= begin_date)\ + .filter(WFC_raw_0.date_obs <= end_date) query_results = query.all() print('\nQuery performed:\n\n{}\n'.format(str(query))) diff --git a/acsql/database/table_definitions/hrc_crj_0.txt b/acsql/database/table_definitions/hrc_crj_0.txt index 113b63a..39bc396 100644 --- a/acsql/database/table_definitions/hrc_crj_0.txt +++ b/acsql/database/table_definitions/hrc_crj_0.txt @@ -44,7 +44,7 @@ BIASLEVD, Float PHOTCORR, String CCDOFSTB, Integer NAXIS, Integer -DATE-OBS, String +DATE_OBS, String FWSOFFST, Integer POSTARG1, Float OSCNTAB, String @@ -75,7 +75,7 @@ ATODCORR, String BADINPDQ, Integer REJ_RATE, Float ATODGND, Float -TIME-OBS, String +TIME_OBS, String DIRIMAGE, String FLASHSTA, String BITPIX, Integer diff --git a/acsql/database/table_definitions/hrc_drz_0.txt b/acsql/database/table_definitions/hrc_drz_0.txt index 4536ac9..2738360 100644 --- a/acsql/database/table_definitions/hrc_drz_0.txt +++ b/acsql/database/table_definitions/hrc_drz_0.txt @@ -51,7 +51,7 @@ PYWCSVER, String PHOTCORR, String D002KERN, String NAXIS, Integer -DATE-OBS, String +DATE_OBS, String FWSOFFST, Integer POSTARG1, Float OSCNTAB, String @@ -542,7 +542,7 @@ D027FVAL, String D032ISCL, String D042VER, String D037GEOM, String -TIME-OBS, String +TIME_OBS, String ORIGIN, String D017OUUN, String D046MASK, String diff --git a/acsql/database/table_definitions/hrc_flt_0.txt b/acsql/database/table_definitions/hrc_flt_0.txt index 29f045d..5914022 100644 --- a/acsql/database/table_definitions/hrc_flt_0.txt +++ b/acsql/database/table_definitions/hrc_flt_0.txt @@ -44,7 +44,7 @@ BIASLEVD, Float PHOTCORR, String CCDOFSTB, Integer NAXIS, Integer -DATE-OBS, String +DATE_OBS, String FWSOFFST, Integer POSTARG1, Float OSCNTAB, String @@ -75,7 +75,7 @@ ATODCORR, String BADINPDQ, Integer REJ_RATE, Float ATODGND, Float -TIME-OBS, String +TIME_OBS, String DIRIMAGE, String FLASHSTA, String BITPIX, Integer diff --git a/acsql/database/table_definitions/hrc_jif_1.txt b/acsql/database/table_definitions/hrc_jif_1.txt index cbf1874..a8b5870 100644 --- a/acsql/database/table_definitions/hrc_jif_1.txt +++ b/acsql/database/table_definitions/hrc_jif_1.txt @@ -67,4 +67,4 @@ GS_GAP, Float GSFAIL, String SGSTAR, String NO_TLM, String -TDF-DOWN, Bool +TDF_DOWN, Bool diff --git a/acsql/database/table_definitions/hrc_jif_2.txt b/acsql/database/table_definitions/hrc_jif_2.txt index 6f514e1..3d46522 100644 --- a/acsql/database/table_definitions/hrc_jif_2.txt +++ b/acsql/database/table_definitions/hrc_jif_2.txt @@ -63,7 +63,7 @@ N_TM_GAP, Integer SLEWING, String TLM_PROB, Bool TM_GAP, Float -TDF-DOWN, Bool +TDF_DOWN, Bool GS_GAP, String GSFAIL, String SGSTAR, String diff --git a/acsql/database/table_definitions/hrc_raw_0.txt b/acsql/database/table_definitions/hrc_raw_0.txt index 187fec7..3cd9519 100644 --- a/acsql/database/table_definitions/hrc_raw_0.txt +++ b/acsql/database/table_definitions/hrc_raw_0.txt @@ -44,7 +44,7 @@ BIASLEVD, Float PHOTCORR, String CCDOFSTB, Integer NAXIS, Integer -DATE-OBS, String +DATE_OBS, String FWSOFFST, Integer POSTARG1, Float OSCNTAB, String @@ -75,7 +75,7 @@ ATODCORR, String BADINPDQ, Integer REJ_RATE, Float ATODGND, Float -TIME-OBS, String +TIME_OBS, String DIRIMAGE, String FLASHSTA, String BITPIX, Integer diff --git a/acsql/database/table_definitions/sbc_drz_0.txt b/acsql/database/table_definitions/sbc_drz_0.txt index 6c58cb7..ac6b683 100644 --- a/acsql/database/table_definitions/sbc_drz_0.txt +++ b/acsql/database/table_definitions/sbc_drz_0.txt @@ -40,7 +40,7 @@ PYWCSVER, String PHOTCORR, String D002KERN, String NAXIS, Integer -DATE-OBS, String +DATE_OBS, String FWSOFFST, Integer POSTARG1, Float OSCNTAB, String @@ -951,7 +951,7 @@ PR_INV_F, String PROCTIME, Float LINENUM, String ASN_ID, String -TIME-OBS, String +TIME_OBS, String PR_INV_L, String ASN_TAB, String PA_V3, Float diff --git a/acsql/database/table_definitions/sbc_flt_0.txt b/acsql/database/table_definitions/sbc_flt_0.txt index cfc5a1c..ae4d3e6 100644 --- a/acsql/database/table_definitions/sbc_flt_0.txt +++ b/acsql/database/table_definitions/sbc_flt_0.txt @@ -36,7 +36,7 @@ LFLTFILE, String PYWCSVER, String PHOTCORR, String NAXIS, Integer -DATE-OBS, String +DATE_OBS, String FWSOFFST, Integer POSTARG1, Float OSCNTAB, String @@ -59,7 +59,7 @@ DRIZCORR, String RA_TARG, Float PA_V3, Float P1_SHAPE, String -TIME-OBS, String +TIME_OBS, String DIRIMAGE, String BITPIX, Integer P1_NPTS, Integer diff --git a/acsql/database/table_definitions/sbc_jif_1.txt b/acsql/database/table_definitions/sbc_jif_1.txt index 6da8818..3ae6cb8 100644 --- a/acsql/database/table_definitions/sbc_jif_1.txt +++ b/acsql/database/table_definitions/sbc_jif_1.txt @@ -65,6 +65,6 @@ TM_GAP, String TLM_PROB, String SLEWING, String N_TM_GAP, String -TDF-DOWN, String +TDF_DOWN, String GSFAIL, String SGSTAR, String diff --git a/acsql/database/table_definitions/sbc_jif_2.txt b/acsql/database/table_definitions/sbc_jif_2.txt index 3c0694f..f9c90e9 100644 --- a/acsql/database/table_definitions/sbc_jif_2.txt +++ b/acsql/database/table_definitions/sbc_jif_2.txt @@ -64,6 +64,6 @@ TLM_PROB, String N_TM_GAP, String GS_GAP, Float TM_GAP, String -TDF-DOWN, String +TDF_DOWN, String GSFAIL, String SGSTAR, String diff --git a/acsql/database/table_definitions/sbc_raw_0.txt b/acsql/database/table_definitions/sbc_raw_0.txt index 14ad2ff..736d066 100644 --- a/acsql/database/table_definitions/sbc_raw_0.txt +++ b/acsql/database/table_definitions/sbc_raw_0.txt @@ -34,7 +34,7 @@ EXPTIME, Float LFLTFILE, String PHOTCORR, String NAXIS, Integer -DATE-OBS, String +DATE_OBS, String FWSOFFST, Integer POSTARG1, Float OSCNTAB, String @@ -57,7 +57,7 @@ DRIZCORR, String RA_TARG, Float PA_V3, Float P1_SHAPE, String -TIME-OBS, String +TIME_OBS, String DIRIMAGE, String BITPIX, Integer P1_NPTS, Integer diff --git a/acsql/database/table_definitions/wfc_crc_0.txt b/acsql/database/table_definitions/wfc_crc_0.txt index 9f63e23..9602006 100644 --- a/acsql/database/table_definitions/wfc_crc_0.txt +++ b/acsql/database/table_definitions/wfc_crc_0.txt @@ -46,7 +46,7 @@ BIASLEVD, Float PHOTCORR, String CCDOFSTB, Integer NAXIS, Integer -DATE-OBS, String +DATE_OBS, String FWSOFFST, Integer POSTARG1, Float OSCNTAB, String @@ -78,7 +78,7 @@ ATODCORR, String BADINPDQ, Integer REJ_RATE, Float ATODGND, Float -TIME-OBS, String +TIME_OBS, String DIRIMAGE, String CTE_VER, String FLASHSTA, String diff --git a/acsql/database/table_definitions/wfc_crj_0.txt b/acsql/database/table_definitions/wfc_crj_0.txt index 3beb01e..2e01373 100644 --- a/acsql/database/table_definitions/wfc_crj_0.txt +++ b/acsql/database/table_definitions/wfc_crj_0.txt @@ -44,7 +44,7 @@ BIASLEVD, Float PHOTCORR, String CCDOFSTB, Integer NAXIS, Integer -DATE-OBS, String +DATE_OBS, String FWSOFFST, Integer POSTARG1, Float OSCNTAB, String @@ -75,7 +75,7 @@ ATODCORR, String BADINPDQ, Integer REJ_RATE, Float ATODGND, Float -TIME-OBS, String +TIME_OBS, String DIRIMAGE, String FLASHSTA, String BITPIX, Integer diff --git a/acsql/database/table_definitions/wfc_drc_0.txt b/acsql/database/table_definitions/wfc_drc_0.txt index d77a336..ca74eef 100644 --- a/acsql/database/table_definitions/wfc_drc_0.txt +++ b/acsql/database/table_definitions/wfc_drc_0.txt @@ -81,7 +81,7 @@ D002KERN, String D008COEF, String D007OUCO, String NAXIS, Integer -DATE-OBS, String +DATE_OBS, String FWSOFFST, Integer POSTARG1, Float OSCNTAB, String diff --git a/acsql/database/table_definitions/wfc_drz_0.txt b/acsql/database/table_definitions/wfc_drz_0.txt index 5e1a9ce..773430b 100644 --- a/acsql/database/table_definitions/wfc_drz_0.txt +++ b/acsql/database/table_definitions/wfc_drz_0.txt @@ -80,7 +80,7 @@ D002KERN, String D008COEF, String D007OUCO, String NAXIS, Integer -DATE-OBS, String +DATE_OBS, String FWSOFFST, Integer POSTARG1, Float OSCNTAB, String @@ -674,7 +674,7 @@ D034OUCO, String D040OUUN, String D033VER, String D033GEOM, String -TIME-OBS, String +TIME_OBS, String CTE_NAME, String D038PIXF, Float OPUS_VER, String diff --git a/acsql/database/table_definitions/wfc_flc_0.txt b/acsql/database/table_definitions/wfc_flc_0.txt index 5020ac7..a176980 100644 --- a/acsql/database/table_definitions/wfc_flc_0.txt +++ b/acsql/database/table_definitions/wfc_flc_0.txt @@ -46,7 +46,7 @@ BIASLEVD, Float PHOTCORR, String CCDOFSTB, Integer NAXIS, Integer -DATE-OBS, String +DATE_OBS, String FWSOFFST, Integer POSTARG1, Float OSCNTAB, String @@ -78,7 +78,7 @@ ATODCORR, String BADINPDQ, Integer REJ_RATE, Float ATODGND, Float -TIME-OBS, String +TIME_OBS, String DIRIMAGE, String CTE_VER, String FLASHSTA, String diff --git a/acsql/database/table_definitions/wfc_flt_0.txt b/acsql/database/table_definitions/wfc_flt_0.txt index 2cef111..9fde26d 100644 --- a/acsql/database/table_definitions/wfc_flt_0.txt +++ b/acsql/database/table_definitions/wfc_flt_0.txt @@ -44,7 +44,7 @@ BIASLEVD, Float PHOTCORR, String CCDOFSTB, Integer NAXIS, Integer -DATE-OBS, String +DATE_OBS, String FWSOFFST, Integer POSTARG1, Float OSCNTAB, String @@ -75,7 +75,7 @@ ATODCORR, String BADINPDQ, Integer REJ_RATE, Float ATODGND, Float -TIME-OBS, String +TIME_OBS, String DIRIMAGE, String FLASHSTA, String BITPIX, Integer diff --git a/acsql/database/table_definitions/wfc_jif_1.txt b/acsql/database/table_definitions/wfc_jif_1.txt index ca31642..6b0faef 100644 --- a/acsql/database/table_definitions/wfc_jif_1.txt +++ b/acsql/database/table_definitions/wfc_jif_1.txt @@ -67,4 +67,4 @@ GSFAIL, String TM_GAP, String NO_TLM, String SLEWING, Bool -TDF-DOWN, Bool +TDF_DOWN, Bool diff --git a/acsql/database/table_definitions/wfc_jif_2.txt b/acsql/database/table_definitions/wfc_jif_2.txt index 3698fc7..40b8719 100644 --- a/acsql/database/table_definitions/wfc_jif_2.txt +++ b/acsql/database/table_definitions/wfc_jif_2.txt @@ -65,6 +65,6 @@ TLM_PROB, String N_TM_GAP, String GS_GAP, Float SGSTAR, String -TDF-DOWN, Bool +TDF_DOWN, Bool NO_TLM, String SLEWING, Bool diff --git a/acsql/database/table_definitions/wfc_jif_3.txt b/acsql/database/table_definitions/wfc_jif_3.txt index 92fda92..80adfef 100644 --- a/acsql/database/table_definitions/wfc_jif_3.txt +++ b/acsql/database/table_definitions/wfc_jif_3.txt @@ -65,6 +65,6 @@ TM_GAP, Float GS_GAP, Float SGSTAR, String GSFAIL, String -TDF-DOWN, Bool +TDF_DOWN, Bool NO_TLM, Bool SLEWING, Bool diff --git a/acsql/database/table_definitions/wfc_jif_4.txt b/acsql/database/table_definitions/wfc_jif_4.txt index d6a4d05..27f934b 100644 --- a/acsql/database/table_definitions/wfc_jif_4.txt +++ b/acsql/database/table_definitions/wfc_jif_4.txt @@ -65,6 +65,6 @@ N_TM_GAP, Integer GS_GAP, Float SGSTAR, String GSFAIL, String -TDF-DOWN, Bool +TDF_DOWN, Bool NO_TLM, Bool SLEWING, Bool diff --git a/acsql/database/table_definitions/wfc_jif_5.txt b/acsql/database/table_definitions/wfc_jif_5.txt index 04c9092..b3e77b5 100644 --- a/acsql/database/table_definitions/wfc_jif_5.txt +++ b/acsql/database/table_definitions/wfc_jif_5.txt @@ -63,7 +63,7 @@ APERTURE, String OBSERVTN, String GUIDEACT, String SGSTAR, String -TDF-DOWN, Bool +TDF_DOWN, Bool GS_GAP, Float NO_TLM, Bool GSFAIL, String diff --git a/acsql/database/table_definitions/wfc_jif_6.txt b/acsql/database/table_definitions/wfc_jif_6.txt index bc07e11..638f6b5 100644 --- a/acsql/database/table_definitions/wfc_jif_6.txt +++ b/acsql/database/table_definitions/wfc_jif_6.txt @@ -46,7 +46,7 @@ TLMFORM, String APER_V2, Float TM_GAP, Float BITPIX, Integer -TDF-DOWN, Bool +TDF_DOWN, Bool XPIXINC, Float OBSERVTN, String LOS_MOON, Float diff --git a/acsql/database/table_definitions/wfc_raw_0.txt b/acsql/database/table_definitions/wfc_raw_0.txt index 7e3d3c9..bfab41a 100644 --- a/acsql/database/table_definitions/wfc_raw_0.txt +++ b/acsql/database/table_definitions/wfc_raw_0.txt @@ -44,7 +44,7 @@ BIASLEVD, Float PHOTCORR, String CCDOFSTB, Integer NAXIS, Integer -DATE-OBS, String +DATE_OBS, String FWSOFFST, Integer POSTARG1, Float OSCNTAB, String @@ -75,7 +75,7 @@ ATODCORR, String BADINPDQ, Integer REJ_RATE, Float ATODGND, Float -TIME-OBS, String +TIME_OBS, String DIRIMAGE, String FLASHSTA, String BITPIX, Integer diff --git a/acsql/ingest/ingest.py b/acsql/ingest/ingest.py index 3087aa0..2c1b1e4 100644 --- a/acsql/ingest/ingest.py +++ b/acsql/ingest/ingest.py @@ -188,12 +188,18 @@ def update_header_table(file_dict, ext): try: for key, value in header.items(): key = key.strip() + + # Switch hypens to underscores + if '-' in key: + key = key.replace('-', '_') + if key in exclude_list or value == "": continue elif key not in TABLE_DEFS[table.lower()]: logging.warning('{}: {} not in {}'\ .format(file_dict['full_rootname'], key, table)) continue + input_dict[key.lower()] = value insert_or_update(table, input_dict) @@ -216,12 +222,13 @@ def update_master_table(rootname_path): """ rootname = os.path.basename(rootname_path)[:-1] + path = rootname_path[-15:] proposid = get_metadata_from_test_files(rootname_path, 'proposid') proposal_type = get_proposal_type(proposid) # Insert a record in the master table data_dict = {'rootname': rootname, - 'path': rootname_path, + 'path': path, 'first_ingest_date': date.today().isoformat(), 'last_ingest_date': date.today().isoformat(), 'detector': get_metadata_from_test_files(rootname_path, diff --git a/acsql/ingest/make_jpeg.py b/acsql/ingest/make_jpeg.py index c3b9574..d7b9ec2 100644 --- a/acsql/ingest/make_jpeg.py +++ b/acsql/ingest/make_jpeg.py @@ -26,7 +26,6 @@ - ``PIL`` """ -import copy import logging import os @@ -57,20 +56,15 @@ def make_jpeg(file_dict): height = data.shape[0] + data2.shape[0] width = data.shape[1] new_array = np.zeros((height, width)) - new_array[0:height/2, :] = data - new_array[height/2:height, :] = data2 + new_array[0:int(height/2), :] = data + new_array[int(height/2):height, :] = data2 data = new_array # Clip the top and bottom 1% of pixels. - sorted_data = copy.copy(data) - sorted_data = sorted_data.ravel() - sorted_data.sort() - top = sorted_data[int(len(sorted_data) * 0.99)] - bottom = sorted_data[int(len(sorted_data) * 0.01)] - top_index = np.where(data > top) - data[top_index] = top - bottom_index = np.where(data < bottom) - data[bottom_index] = bottom + top = np.percentile(data, 99) + data[data > top] = top + bottom = np.percentile(data, 1) + data[data < bottom] = bottom # Scale the data. data = data - data.min() diff --git a/acsql/website/data_containers.py b/acsql/website/data_containers.py index a750e7c..4f4ea3b 100644 --- a/acsql/website/data_containers.py +++ b/acsql/website/data_containers.py @@ -93,10 +93,9 @@ def _get_metadata_from_database(data_dict): table = getattr(database_interface, '{}_raw_0'.format(detector)) result = session.query( table.aperture, table.exptime, table.filter1, table.filter2, - table.targname, getattr(table, 'date-obs'), - getattr(table, 'time-obs'), table.expstart, table.expflag, - table.quality, table.ra_targ, table.dec_targ, table.pr_inv_f, - table.pr_inv_l).filter(table.rootname == rootname).one() + table.targname, table.date_obs, table.time_obs, table.expstart, + table.expflag, table.quality, table.ra_targ, table.dec_targ, + table.pr_inv_f, table.pr_inv_l).filter(table.rootname == rootname).one() result = [item for item in result] result.append(detector) results.append(result) diff --git a/acsql/website/form_options.py b/acsql/website/form_options.py index 4d0afc5..2573e08 100644 --- a/acsql/website/form_options.py +++ b/acsql/website/form_options.py @@ -41,7 +41,7 @@ ('proposal_type','Proposal Type'), ('pr_inv_l','PI Last Name'), ('pr_inv_f','PI First Name'), ('proposid','Proposal ID'), ('filter1','Filter1'), ('filter2','Filter2'), ('aperture','Aperture'), ('expstart','Expstart'), - ('date-obs','Date of Observation'), ('time-obs','Time of Observation'), + ('date_obs','Date of Observation'), ('time_obs','Time of Observation'), ('targname','Target Name'), ('ra_targ','Target RA'), ('dec_targ','Target Dec'), ('obstype','Observation Type'), ('obsmode','Observation Mode'), ('subarray','Subarray'), ('imagetyp', 'Image Type'), ('asn_id','Association ID')]