Data Transformation Functions

from gluestick.etl_utils import clean_convert

# Clean dictionary
cleaned_dict = clean_convert({
    'name': 'John',
    'age': None,
    'scores': [10, None, 20]
})

# Clean nested structures
cleaned_nested = clean_convert({
    'user': {
        'name': 'John',
        'details': [
            {'score': 10},
            {'score': None}
        ]
    }
})

from gluestick.etl_utils import map_fields

# Basic field mapping
mapping = {
    'user_name': 'name',
    'user_age': 'age'
}
result = map_fields(row, mapping)

# Nested mapping
mapping = {
    'user': {
        'fullName': 'name',
        'age': 'age'
    },
    'scores': ['test_scores']
}
result = map_fields(row, mapping)

from gluestick.pandas_utils import rename

# Basic column renaming
new_df = rename(df, {
    'old_name': 'new_name',
    'previous': 'current'
})

# Select specific columns
new_df = rename(df, ['col1', 'col2', 'col3'])

from gluestick.etl_utils import localize_datetime

# Localize single column
df['timestamp'] = localize_datetime(df, 'timestamp')

# Process multiple columns
for col in ['created_at', 'updated_at']:
    df[col] = localize_datetime(df, col)

from gluestick.singer import deep_convert_datetimes

# Convert nested structure
converted = deep_convert_datetimes({
    'created': datetime.now(),
    'items': [
        {'date': datetime.now()},
        {'date': datetime.now()}
    ]
})

# Convert list of records
converted_list = deep_convert_datetimes([
    {'timestamp': datetime.now()},
    {'timestamp': datetime.now()}
])

from gluestick.etl_utils import exception

try:
    # Your ETL code here
    process_data()
except ValueError as e:
    exception(e, ROOT_DIR, "Error processing data")

# With additional context
try:
    transform_data()
except Exception as e:
    exception(
        e,
        ROOT_DIR,
        f"Failed processing stream {stream_name}"
    )

import gluestick as gs
from datetime import datetime

def process_timestamps(df):
    # Localize timezone columns
    datetime_cols = ['created_at', 'updated_at']
    for col in datetime_cols:
        df[col] = gs.localize_datetime(df, col)
    
    # Process nested data with timestamps
    df['metadata'] = df['metadata'].apply(gs.deep_convert_datetimes)
    
    return df

# Clean and map data with timestamps
def transform_data(input_data):
    # First clean None values and convert dates
    cleaned = gs.clean_convert(input_data)
    
    # Define mapping with timestamp fields
    mapping = {
        'user': {
            'created': 'created_at',
            'modified': 'updated_at'
        }
    }
    
    # Map and process fields
    return gs.map_fields(cleaned, mapping)