Example Reduction

import os
os.environ['OPENTSDB_PYTHON_METRICS_TEST_MODE'] = 'True'

from banzai.calibrations import make_master_calibrations
import requests
from banzai_floyds import settings
import banzai.dbs
from banzai.utils.stage_utils import run_pipeline_stages
import logging
from banzai.logs import set_log_level
from glob import glob

import importlib.resources
import banzai_floyds.dbs
from astropy.io import ascii, fits
from banzai.utils.fits_utils import download_from_s3

os.environ['DB_ADDRESS'] = 'sqlite:///test_data/test.db'

settings.processed_path= os.path.join(os.getcwd(), 'test_data')
settings.fpack=True
settings.db_address = os.environ['DB_ADDRESS']
settings.reduction_level = 91
settings.ARCHIVE_API_ROOT = 'https://archive-api.lco.global/'
settings.RAW_DATA_API_ROOT = 'https://archive-api.lco.global/'
settings.RAW_DATA_FRAME_URL = 'https://archive-api.lco.global/frames'
settings.ARCHIVE_FRAME_URL = 'https://archive-api.lco.global/frames'

# set up the context object.
import banzai.main
context = banzai.main.parse_args(settings, parse_system_args=False)

os.makedirs('test_data', exist_ok=True)

banzai.dbs.create_db(os.environ["DB_ADDRESS"])
os.system(f'banzai_add_site --site ogg --latitude 20.7069444444 --longitude -156.258055556 --elevation 3065 --timezone -10 --db-address={os.environ["DB_ADDRESS"]}')
os.system(f'banzai_add_site --site coj --latitude -31.272932 --longitude 149.070648 --elevation 1116 --timezone 10 --db-address={os.environ["DB_ADDRESS"]}')
os.system(f'banzai_add_instrument --site ogg --camera en06 --name floyds01 --instrument-type 2m0-FLOYDS-SciCam --db-address={os.environ["DB_ADDRESS"]}')
os.system(f'banzai_add_instrument --site coj --camera en12 --name floyds02 --instrument-type 2m0-FLOYDS-SciCam --db-address={os.environ["DB_ADDRESS"]}')
banzai_floyds.dbs.ingest_standards(os.environ["DB_ADDRESS"])

skyflat_files = ascii.read(os.path.join(importlib.resources.files('banzai_floyds.tests'), 'data', 'test_skyflat.dat'))
for skyflat in skyflat_files:
    skyflat_info = dict(skyflat)
    context = banzai.main.parse_args(settings, parse_system_args=False)
    skyflat_hdu = fits.open(download_from_s3(skyflat_info, context))

    # Munge the data to be OBSTYPE SKYFLAT
    skyflat_hdu['SCI'].header['OBSTYPE'] = 'SKYFLAT'
    skyflat_name = skyflat_info["filename"].replace("x00.fits", "f00.fits")
    filename = os.path.join('test_data', f'{skyflat_name}')
    skyflat_hdu.writeto(filename, overwrite=True)
    skyflat_hdu.close()
    run_pipeline_stages([{'path': filename}], context)

DATA_FILELIST = os.path.join(importlib.resources.files('banzai_floyds.tests'), 'data', 'test_data.dat')
test_data = ascii.read(DATA_FILELIST)

for row in test_data:
    if 'a00.fits' in row['filename']:
        archive_record = requests.get(f'{context.ARCHIVE_FRAME_URL}/{row["frameid"]}').json()
        archive_record['frameid'] = archive_record['id']
        run_pipeline_stages([archive_record], context)

for row in test_data:
    if 'w00.fits' in row['filename']:
        archive_record = requests.get(f'{context.ARCHIVE_FRAME_URL}/{row["frameid"]}').json()
        archive_record['frameid'] = archive_record['id']
        run_pipeline_stages([archive_record], context)

coj_floyds = banzai.dbs.get_instruments_at_site('coj', settings.db_address)[0]
ogg_floyds = banzai.dbs.get_instruments_at_site('ogg', settings.db_address)[0]
make_master_calibrations(ogg_floyds, 'LAMPFLAT', '2020-01-01', '2021-01-01', context)
make_master_calibrations(coj_floyds, 'LAMPFLAT', '2020-01-01', '2021-01-01', context)

for row in test_data:
    if 'e00.fits' in row['filename'] and ('BD+284211' in row['object'] or 'FEIGE110' in row['OBJECT']):
        archive_record = requests.get(f'{context.ARCHIVE_FRAME_URL}/{row["frameid"]}').json()
        archive_record['frameid'] = archive_record['id']
        run_pipeline_stages([archive_record], context)

for row in test_data:
    if 'e00.fits' in row['filename'] and not ('BD+284211' in row['object'] or 'FEIGE110' in row['OBJECT']):
        archive_record = requests.get(f'{context.ARCHIVE_FRAME_URL}/{row["frameid"]}').json()
        archive_record['frameid'] = archive_record['id']
        run_pipeline_stages([archive_record], context)