import os
import requests
import pandas as pd
import numpy as np
def load_data():
csv_path = 'macro_growth_data.csv'
if not os.path.exists(csv_path):
csv_path = 'posts/macro_growth_data.csv'
if not os.path.exists(csv_path):
csv_path = '../posts/macro_growth_data.csv'
if os.path.exists(csv_path):
df = pd.read_csv(csv_path)
df['date'] = pd.to_datetime(df['date'])
return df
# Fallback: API fetch if CSV does not exist
env_candidates = ['.env', '../.env', '../../.env']
api_key = None
for p in env_candidates:
if os.path.exists(p):
with open(p) as f:
for line in f:
if line.strip().startswith('FRED_API_KEY='):
api_key = line.strip().split('=', 1)[1].strip('\"\'')
break
if api_key:
break
if not api_key:
api_key = os.getenv('FRED_API_KEY')
if not api_key:
raise ValueError("FRED_API_KEY not found.")
indicators = [
{'symbol': 'GDPC1', 'name': 'Real Gross Domestic Product', 'category': 'growth', 'tendency': 'lagging', 'freq': 'Q'},
{'symbol': 'PAYEMS', 'name': 'All Employees, Total Nonfarm', 'category': 'growth', 'tendency': 'coincident', 'freq': 'M'},
{'symbol': 'INDPRO', 'name': 'Industrial Production Index', 'category': 'growth', 'tendency': 'leading', 'freq': 'M'},
{'symbol': 'HOUST', 'name': 'New Privately-Owned Housing Units Started', 'category': 'growth', 'tendency': 'leading', 'freq': 'M'}
]
url = 'https://api.stlouisfed.org/fred/series/observations'
all_dfs = []
for ind in indicators:
sym = ind['symbol']
params = {
'series_id': sym,
'api_key': api_key,
'file_type': 'json',
'output_type': 1,
'realtime_start': '1776-07-04',
'observation_start': '1959-01-01',
'units': 'lin',
'limit': 100000
}
res = requests.get(url, params=params, timeout=60).json()
raw_df = pd.DataFrame(res.get('observations', []))
raw_df['date'] = pd.to_datetime(raw_df['date'])
raw_df['realtime_start'] = pd.to_datetime(raw_df['realtime_start'])
raw_df['realtime_end'] = pd.to_datetime(raw_df['realtime_end'])
raw_df['value'] = pd.to_numeric(raw_df['value'], errors='coerce')
max_lag = 150 if ind['freq'] == 'Q' else 90
latest_rows = raw_df[raw_df['realtime_end'] == pd.to_datetime('9999-12-31')].sort_values('date').drop_duplicates('date')
latest_dict = dict(zip(latest_rows['date'], latest_rows['value']))
sorted_df = raw_df.sort_values(['date', 'realtime_start'])
first_releases = sorted_df.drop_duplicates('date', keep='first').copy()
first_releases['lag_days'] = (first_releases['realtime_start'] - first_releases['date']).dt.days
valid_flashes = first_releases[first_releases['lag_days'] <= max_lag].copy()
lookup = {}
for d, grp in raw_df.groupby('date'):
lookup[d] = grp[['realtime_start', 'realtime_end', 'value']].values
date_offset = pd.DateOffset(years=1)
sub_results = []
for _, r in valid_flashes.iterrows():
d = r['date']
f_date = r['realtime_start']
f_val = r['value']
prior_d = d - date_offset
if prior_d not in lookup:
continue
matched = [v for (s, e, v) in lookup[prior_d] if s <= f_date <= e and not np.isnan(v)]
if not matched:
continue
prior_f_val = matched[0]
if prior_f_val == 0 or np.isnan(prior_f_val) or np.isnan(f_val):
continue
flash_yoy = ((f_val - prior_f_val) / abs(prior_f_val)) * 100.0
final_val = latest_dict.get(d, np.nan)
final_prior_val = latest_dict.get(prior_d, np.nan)
if np.isnan(final_val) or np.isnan(final_prior_val) or final_prior_val == 0:
continue
final_yoy = ((final_val - final_prior_val) / abs(final_prior_val)) * 100.0
rev = final_yoy - flash_yoy
sign_flip = 1 if (flash_yoy * final_yoy < 0) else 0
sub_results.append({
'date': d.strftime('%Y-%m-%d'),
'symbol': sym,
'name': ind['name'],
'category': ind['category'],
'tendency': ind['tendency'],
'frequency': ind['freq'],
'flash_release_date': f_date.strftime('%Y-%m-%d'),
'flash_yoy': round(flash_yoy, 4),
'final_yoy': round(final_yoy, 4),
'revision': round(rev, 4),
'abs_revision': round(abs(rev), 4),
'sign_flip': sign_flip
})
all_dfs.append(pd.DataFrame(sub_results))
combined = pd.concat(all_dfs, ignore_index=True)
combined['date'] = pd.to_datetime(combined['date'])
usrec_params = {'series_id': 'USREC', 'api_key': api_key, 'file_type': 'json', 'observation_start': '1959-01-01', 'limit': 100000}
usrec_res = requests.get(url, params=usrec_params).json()
usrec_df = pd.DataFrame(usrec_res.get('observations', []))
usrec_df['date'] = pd.to_datetime(usrec_df['date'])
usrec_df['usrec'] = pd.to_numeric(usrec_df['value'], errors='coerce').fillna(0).astype(int)
merged = pd.merge(combined, usrec_df[['date', 'usrec']], on='date', how='left')
merged['usrec'] = merged['usrec'].fillna(0).astype(int)
merged['date'] = merged['date'].dt.strftime('%Y-%m-%d')
merged = merged[merged['date'] >= '1960-01-01'].sort_values(['symbol', 'date'])
merged.to_csv('macro_growth_data.csv', index=False)
merged['date'] = pd.to_datetime(merged['date'])
return merged
df = load_data()
df.head()
