Load DataFrame
The @sql.load_dataframe decorator and DataFrameLoadOperator load a pandas DataFrame directly into a database table.
Operator Usage
import pandas as pd
from airsql.operators import DataFrameLoadOperator
from airsql import Table
df = pd.DataFrame({
'id': [1, 2, 3],
'name': ['Alice', 'Bob', 'Charlie'],
'value': [100, 200, 300],
})
load_task = DataFrameLoadOperator(
task_id='load_dataframe',
dataframe=df,
output_table=Table('postgres_conn', 'staging.loaded_data'),
if_exists='replace',
)
load_task.execute(context)
Parameters
| Parameter |
Type |
Required |
Description |
task_id |
str |
Yes |
Unique task identifier |
dataframe |
pd.DataFrame |
Yes |
Pandas DataFrame to load |
output_table |
Table |
Yes |
Destination table reference |
if_exists |
str |
No |
Behavior: 'append', 'replace', 'fail', 'truncate' |
timestamp_column |
str |
No |
Column to populate with current timestamp |
dry_run_flag |
bool |
No |
If True, simulate without writing |
**kwargs |
Any |
No |
Additional operator kwargs |
Decorator Usage
Function Returns DataFrame
from airsql import sql, Table
import pandas as pd
@sql.load_dataframe(
output_table=Table('postgres_conn', 'analytics.user_summary'), if_exists='replace'
)
def create_user_summary():
return pd.DataFrame({
'user_id': [1, 2, 3],
'name': ['Alice', 'Bob', 'Charlie'],
'total_orders': [10, 5, 8],
})
load_task = create_user_summary()
Direct DataFrame Loading
import pandas as pd
df = pd.DataFrame({'id': [1, 2], 'value': [10, 20]})
@sql.load_dataframe(output_table=Table('postgres_conn', 'staging.data'), dataframe=df)
def load_existing_data():
pass
Examples
Replace Table Content
@sql.load_dataframe(
output_table=Table('postgres_conn', 'reports.summary'), if_exists='replace'
)
def generate_summary():
return pd.DataFrame({
'metric': ['users', 'orders', 'revenue'],
'value': [1000, 500, 25000],
})
Append to Existing Table
@sql.load_dataframe(
output_table=Table('postgres_conn', 'staging.events'), if_exists='append'
)
def add_events():
return pd.DataFrame({
'event_id': [101, 102],
'event_type': ['click', 'purchase'],
'timestamp': [pd.Timestamp.now(), pd.Timestamp.now()],
})
With Timestamp Column
@sql.load_dataframe(
output_table=Table('postgres_conn', 'audit.changes'),
if_exists='append',
timestamp_column='loaded_at',
)
def track_changes():
return pd.DataFrame({'entity_id': [1, 2], 'change_type': ['update', 'create']})
Dry Run Mode
@sql.load_dataframe(
output_table=Table('postgres_conn', 'test.data'), if_exists='replace', dry_run=True
)
def test_load():
return pd.DataFrame({'col': [1, 2, 3]})
if_exists Options
| Value |
Behavior |
append |
Add data to existing table |
replace |
Drop and recreate table |
truncate |
Truncate table, preserve structure |
fail |
Raise error if table exists |
Temporary Tables
temp_table = Table('postgres_conn', 'temp.staging', temporary=True)
@sql.load_dataframe(output_table=temp_table, if_exists='replace')
def create_temp_data():
return pd.DataFrame({'id': [1, 2, 3]})