import numpy as np

# 1-dimensional array
arr1d = np.array([1, 2, 3, 4, 5])
print("1D Array:", arr1d)
print("Type:", type(arr1d))
print("Shape:", arr1d.shape)
print("Dimension (ndim):", arr1d.ndim)
print("Data Type (dtype):", arr1d.dtype)

# 2-dimensional array (matrix)
arr2d = np.array([[1, 2, 3], [4, 5, 6]])
print("\n2D Array:\n", arr2d)
print("Shape:", arr2d.shape)
print("Dimension (ndim):", arr2d.ndim)

# Array of zeros
zeros_arr = np.zeros((2, 3))
print("\nZeros Array:\n", zeros_arr)

# Array of ones
ones_arr = np.ones((3, 2))
print("\nOnes Array:\n", ones_arr)

# Array filled with a specific value
full_arr = np.full((2, 2), 7)
print("\nFull Array (with 7s):\n", full_arr)

# Array with a range of values
range_arr = np.arange(0, 10, 2) # start, stop (exclusive), step
print("\nArange Array:", range_arr)

# Array with linearly spaced values
linspace_arr = np.linspace(0, 1, 5) # start, stop (inclusive), num_elements
print("\nLinspace Array:", linspace_arr)

# Identity matrix
identity_arr = np.eye(3)
print("\nIdentity Array:\n", identity_arr)

arr = np.array([[1, 2, 3], [4, 5, 6]])
print("\nArray Attributes:")
print("Number of dimensions (ndim):", arr.ndim)
print("Shape (rows, columns):", arr.shape)
print("Total elements (size):", arr.size)
print("Data type of elements (dtype):", arr.dtype)
print("Size of each element in bytes (itemsize):", arr.itemsize)

int_arr = np.array([1, 2, 3], dtype=np.int32)
print("\nInteger Array (int32):", int_arr, int_arr.dtype)

float_arr = np.array([1, 2, 3], dtype=np.float64)
print("Float Array (float64):", float_arr, float_arr.dtype)

arr = np.array([10, 20, 30, 40, 50])
print("\nOriginal 1D array:", arr)
print("Element at index 0:", arr[0])
print("Element at index -1 (last):", arr[-1])

arr2d = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
print("\nOriginal 2D array:\n", arr2d)
print("Element at (0, 0):", arr2d[0, 0])
print("Element at (1, 2):", arr2d[1, 2])

arr = np.array([10, 20, 30, 40, 50, 60, 70])
print("\nOriginal 1D array for slicing:", arr)
print("Elements from index 1 to 4 (exclusive):", arr[1:5])
print("Elements from start to index 3 (exclusive):", arr[:4])
print("Elements from index 2 to end:", arr[2:])
print("Every other element:", arr[::2])
print("Reversed array:", arr[::-1])

arr2d = np.array([[1, 2, 3, 4],
                  [5, 6, 7, 8],
                  [9, 10, 11, 12]])
print("\nOriginal 2D array for slicing:\n", arr2d)
# All rows, columns 1 to 3 (exclusive)
print("All rows, columns 1 and 2:\n", arr2d[:, 1:3])
# Rows 0 to 1 (exclusive), all columns
print("Row 0, all columns:\n", arr2d[0:1, :]) # Note: [0:1] keeps it 2D
print("Row 0 (as 1D array):\n", arr2d[0, :])
# Subarray from row 1, col 0 to row 2, col 1
print("Subarray from [1,0] to [2,1]:\n", arr2d[1:3, 0:2])

arr = np.array([100, 200, 300, 400, 500])
indices = np.array([0, 2, 4])
print("\nOriginal 1D array:", arr)
print("Elements at specific indices (0, 2, 4):", arr[indices])

arr2d = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
print("\nOriginal 2D array:\n", arr2d)
# Select rows 0 and 2
print("Rows 0 and 2:\n", arr2d[[0, 2]])
# Select specific elements at (0,0), (1,1), (2,0)
print("Specific elements using fancy indexing:", arr2d[[0, 1, 2], [0, 1, 0]])

arr = np.array([1, 5, 2, 8, 3, 9, 4])
print("\nOriginal 1D array:", arr)
# Select elements greater than 4
bool_mask = (arr > 4)
print("Boolean mask (arr > 4):", bool_mask)
print("Elements greater than 4:", arr[bool_mask])

arr2d = np.array([[10, 20, 30],
                  [40, 50, 60],
                  [70, 80, 90]])
print("\nOriginal 2D array:\n", arr2d)
# Select elements greater than 50
print("Elements greater than 50:\n", arr2d[arr2d > 50])

arr = np.arange(1, 10)
print("\nOriginal array:", arr)
reshaped_arr = arr.reshape((3, 3))
print("Reshaped to 3x3:\n", reshaped_arr)

# Using -1 to automatically infer a dimension
reshaped_arr_auto = arr.reshape((3, -1))
print("Reshaped to 3x? (auto infer):\n", reshaped_arr_auto)

flattened_arr = reshaped_arr.flatten()
print("Flattened array:", flattened_arr)

arr = np.array([[1, 2, 3], [4, 5, 6]])
print("\nOriginal 2D array:\n", arr)
transposed_arr = arr.T
print("Transposed array:\n", transposed_arr)

arr1 = np.array([1, 2, 3])
arr2 = np.array([4, 5, 6])
print("\nArray 1:", arr1)
print("Array 2:", arr2)

# Concatenate 1D arrays
concatenated_1d = np.concatenate((arr1, arr2))
print("Concatenated 1D:", concatenated_1d)

mat1 = np.array([[1, 2], [3, 4]])
mat2 = np.array([[5, 6], [7, 8]])
print("\nMatrix 1:\n", mat1)
print("Matrix 2:\n", mat2)

# Vertical Stack
vstacked = np.vstack((mat1, mat2))
print("Vertically stacked:\n", vstacked)

# Horizontal Stack
hstacked = np.hstack((mat1, mat2))
print("Horizontally stacked:\n", hstacked)

# Splitting arrays
arr_to_split = np.arange(12).reshape(3, 4)
print("\nArray to split:\n", arr_to_split)
h_split = np.hsplit(arr_to_split, 2) # Split into 2 columns
print("Horizontal split (2 parts):\n", h_split)
v_split = np.vsplit(arr_to_split, 3) # Split into 3 rows
print("Vertical split (3 parts):\n", v_split)

arr = np.array([1, 2, 3])
print("\nOriginal 1D array:", arr, "Shape:", arr.shape)

# Add a new axis (make it a row vector)
row_vec = arr[np.newaxis, :]
print("As row vector:", row_vec, "Shape:", row_vec.shape)

# Add a new axis (make it a column vector)
col_vec = arr[:, np.newaxis]
print("As column vector:\n", col_vec, "Shape:", col_vec.shape)

squeezed_arr = np.squeeze(row_vec)
print("Squeezed row vector:", squeezed_arr, "Shape:", squeezed_arr.shape)

arr1 = np.array([1, 2, 3])
arr2 = np.array([4, 5, 6])

print("\nElement-wise addition:", arr1 + arr2)
print("Element-wise multiplication:", arr1 * arr2)
print("Element-wise division:", arr1 / arr2)
print("Element-wise exponentiation:", arr1 ** 2)
print("Square root of arr2:", np.sqrt(arr2))
print("Sine of arr1:", np.sin(arr1))

arr = np.array([[1, 2, 3], [4, 5, 6]])
scalar = 10
print("\nOriginal array:\n", arr)
print("Array + scalar:\n", arr + scalar) # Scalar is broadcast across the array

vec = np.array([100, 200, 300]) # shape (3,)
print("\nOriginal array:\n", arr)
print("Vector for broadcasting:", vec)
# vec (100, 200, 300) is broadcast across each row of arr
print("Array + vector:\n", arr + vec)

# Example with explicit newaxis for broadcasting clarity
col_vec = np.array([[10], [20]]) # shape (2,1)
print("\nOriginal array:\n", arr)
print("Column vector for broadcasting:\n", col_vec)
# col_vec (2,1) is broadcast across columns of arr (2,3)
print("Array + column vector:\n", arr + col_vec)

A = np.array([[1, 2], [3, 4]])
B = np.array([[5, 6], [7, 8]])

print("\nMatrix A:\n", A)
print("Matrix B:\n", B)

# Matrix multiplication
print("Matrix multiplication (A @ B):\n", A @ B)
print("Matrix multiplication (np.dot(A, B)):\n", np.dot(A, B))

# Inverse of A
try:
    A_inv = np.linalg.inv(A)
    print("Inverse of A:\n", A_inv)
    print("A dot A_inv:\n", A @ A_inv) # Should be identity matrix
except np.linalg.LinAlgError as e:
    print(f"Error calculating inverse: {e}")

# Determinant of A
print("Determinant of A:", np.linalg.det(A))

arr = np.array([[1, 2, 3], [4, 5, 6]])
print("\nOriginal array:\n", arr)

print("Sum of all elements:", np.sum(arr))
print("Minimum of all elements:", np.min(arr))
print("Mean of all elements:", np.mean(arr))

print("\nSum along axis 0 (columns sum down):\n", np.sum(arr, axis=0))
print("Mean along axis 1 (rows mean across):\n", np.mean(arr, axis=1))

print("Standard deviation of all elements:", np.std(arr))

np.random.seed(42) # For reproducibility

print("\nRandom numbers (uniform in [0,1)):", np.random.rand(3))
print("Random numbers (standard normal):", np.random.randn(2, 2))
print("Random integers (0-9, size 5):", np.random.randint(0, 10, size=5))
print("Random numbers (normal, mean=5, std=1, size=3):", np.random.normal(5, 1, size=3))

import time

# Pure Python list sum
list_data = list(range(10_000_000))
start_time = time.time()
list_sum = sum(list_data)
end_time = time.time()
print(f"\nPython list sum took: {end_time - start_time:.6f} seconds")

# NumPy array sum
numpy_data = np.arange(10_000_000)
start_time = time.time()
numpy_sum = np.sum(numpy_data)
end_time = time.time()
print(f"NumPy array sum took: {end_time - start_time:.6f} seconds")

import pandas as pd
import numpy as np

# From a list
s1 = pd.Series([1, 2, 3, 4, 5])
print("Series from list:\n", s1)
print("Index:", s1.index)
print("Values:", s1.values)

# From a list with a custom index
s2 = pd.Series([10, 20, 30], index=['a', 'b', 'c'])
print("\nSeries with custom index:\n", s2)

# From a dictionary
data = {'apple': 100, 'banana': 150, 'cherry': 200}
s3 = pd.Series(data)
print("\nSeries from dictionary:\n", s3)

# From a scalar value (index must be provided)
s4 = pd.Series(5, index=['x', 'y', 'z'])
print("\nSeries from scalar:\n", s4)

# From a dictionary of lists/arrays
data = {
    'Name': ['Alice', 'Bob', 'Charlie', 'David'],
    'Age': [25, 30, 35, 40],
    'City': ['New York', 'London', 'Paris', 'Tokyo']
}
df1 = pd.DataFrame(data)
print("DataFrame from dictionary of lists:\n", df1)
print("Columns:", df1.columns)
print("Index:", df1.index)
print("Shape:", df1.shape)
print("Data Types:\n", df1.dtypes)

# From a list of dictionaries
data_list_dict = [
    {'Name': 'Eve', 'Age': 22, 'City': 'Rome'},
    {'Name': 'Frank', 'Age': 28, 'City': 'Berlin'}
]
df2 = pd.DataFrame(data_list_dict)
print("\nDataFrame from list of dictionaries:\n", df2)

# From a NumPy array (with optional columns and index)
numpy_data = np.random.rand(4, 3)
df3 = pd.DataFrame(numpy_data, columns=['ColA', 'ColB', 'ColC'], index=['r1', 'r2', 'r3', 'r4'])
print("\nDataFrame from NumPy array:\n", df3)

# Example: Creating a dummy CSV file to demonstrate reading
csv_data = """id,name,age,city,salary
1,Alice,25,New York,60000
2,Bob,30,London,75000
3,Charlie,NaN,Paris,90000
4,David,40,Tokyo,NaN
5,Eve,28,Berlin,65000
"""
with open("sample_data.csv", "w") as f:
    f.write(csv_data)

# Reading a CSV file
df_csv = pd.read_csv("sample_data.csv")
print("\nDataFrame loaded from CSV:\n", df_csv)

# To clean up the dummy file
import os
os.remove("sample_data.csv")

print("\n--- DataFrame Inspection (df_csv) ---")
print("Head (first 3 rows):\n", df_csv.head(3))
print("\nTail (last 2 rows):\n", df_csv.tail(2))
print("\nDataFrame Info:")
df_csv.info()
print("\nDescriptive Statistics:\n", df_csv.describe())
print("\nShape:", df_csv.shape)
print("Columns:", df_csv.columns)
print("Index:", df_csv.index)

# Select a single column (returns a Series)
ages = df_csv['age']
print("\n'age' column (Series):\n", ages)
print("Type of 'age' column:", type(ages))

# Select multiple columns (returns a DataFrame)
name_city = df_csv[['name', 'city']]
print("\n'name' and 'city' columns (DataFrame):\n", name_city)
print("Type of ['name', 'city'] columns:", type(name_city))

# Selecting rows by label using .loc
df_loc_indexed = df_csv.set_index('name') # Set 'name' as index for this example
print("\nDataFrame with 'name' as index:\n", df_loc_indexed)

print("\nRow 'Alice' using .loc:\n", df_loc_indexed.loc['Alice'])
print("\nRows 'Alice' and 'David' using .loc:\n", df_loc_indexed.loc[['Alice', 'David']])
print("\nRows 'Alice' to 'Charlie' (inclusive) using .loc:\n", df_loc_indexed.loc['Alice':'Charlie'])

# Selecting rows by integer position using .iloc
print("\nRow at integer position 0 using .iloc:\n", df_csv.iloc[0])
print("\nRows at integer positions 1 and 3 using .iloc:\n", df_csv.iloc[[1, 3]])
print("\nRows from integer position 0 to 2 (exclusive) using .iloc:\n", df_csv.iloc[0:2])

# Using .loc with both row and column labels
# Select 'age' and 'city' for 'Alice'
print("\n'age' and 'city' for 'Alice' using .loc:", df_loc_indexed.loc['Alice', ['age', 'city']])
# Select all columns for 'Bob' to 'Eve'
print("\nAll columns for 'Bob' to 'Eve' using .loc:\n", df_loc_indexed.loc['Bob':'Eve', :])

# Using .iloc with both row and column integer positions
# Select row 0, column 1 (name)
print("\nRow 0, column 1 (name) using .iloc:", df_csv.iloc[0, 1])
# Select rows 0 and 1, columns 2 and 3 (age, city)
print("\nRows 0 and 1, columns 2 and 3 (age, city) using .iloc:\n", df_csv.iloc[[0, 1], [2, 3]])

print("\nOriginal DataFrame:\n", df_csv)

# Select people older than 30
df_older_than_30 = df_csv[df_csv['age'] > 30]
print("\nPeople older than 30:\n", df_older_than_30)

# Select people from 'London' OR 'Berlin'
df_london_berlin = df_csv[(df_csv['city'] == 'London') | (df_csv['city'] == 'Berlin')]
print("\nPeople from London or Berlin:\n", df_london_berlin)

# Select people with age > 25 AND city is 'New York'
df_filtered_complex = df_csv[(df_csv['age'] > 25) & (df_csv['city'] == 'New York')]
print("\nPeople > 25 from New York:\n", df_filtered_complex)

# Using .isin() for multiple values
df_specific_cities = df_csv[df_csv['city'].isin(['New York', 'Tokyo'])]
print("\nPeople from New York or Tokyo using .isin():\n", df_specific_cities)

print("\nDataFrame with NaNs:\n", df_csv)
print("\nIs Null (Boolean DataFrame):\n", df_csv.isnull())
print("\nMissing values per column:\n", df_csv.isnull().sum())
print("\nTotal missing values:", df_csv.isnull().sum().sum())

df_dropna_rows = df_csv.dropna() # Drops rows with ANY NaN
print("\nDataFrame after dropping rows with ANY NaN:\n", df_dropna_rows)

df_dropna_all = df_csv.dropna(how='all') # Drops rows where ALL values are NaN
print("\nDataFrame after dropping rows with ALL NaN (no change here):\n", df_dropna_all)

df_dropna_columns = df_csv.dropna(axis=1) # Drops columns with ANY NaN
print("\nDataFrame after dropping columns with ANY NaN:\n", df_dropna_columns)

df_dropna_threshold = df_csv.dropna(thresh=4) # Keep rows with at least 4 non-NaN values
print("\nDataFrame after dropping rows with less than 4 non-NaNs:\n", df_dropna_threshold)

# Fill with a specific value
df_filled_value = df_csv.fillna(0)
print("\nDataFrame after filling NaNs with 0:\n", df_filled_value)

# Fill 'age' with its mean, 'salary' with its median
df_filled_mean_median = df_csv.copy() # Work on a copy
mean_age = df_filled_mean_median['age'].mean()
median_salary = df_filled_mean_median['salary'].median()

df_filled_mean_median['age'] = df_filled_mean_median['age'].fillna(mean_age)
df_filled_mean_median['salary'] = df_filled_mean_median['salary'].fillna(median_salary)
print(f"\nMean age: {mean_age:.2f}, Median salary: {median_salary:.2f}")
print("DataFrame after filling 'age' with mean, 'salary' with median:\n", df_filled_mean_median)

# Forward fill (ffill) - propagates last valid observation forward to next valid
df_ffill = df_csv.fillna(method='ffill')
print("\nDataFrame after forward fill:\n", df_ffill)

# Backward fill (bfill) - propagates next valid observation backward to next valid
df_bfill = df_csv.fillna(method='bfill')
print("\nDataFrame after backward fill:\n", df_bfill)

df_renamed = df_csv.rename(columns={'id': 'employee_id', 'city': 'location'})
print("\nDataFrame after renaming columns:\n", df_renamed)

df_dtypes = pd.DataFrame({'A': [1, 2, 3], 'B': ['4', '5', '6']})
print("\nOriginal dtypes:\n", df_dtypes.dtypes)
df_dtypes['B'] = df_dtypes['B'].astype(int)
print("\nNew dtypes after converting 'B' to int:\n", df_dtypes.dtypes)

# Converting to datetime
df_dates = pd.DataFrame({'date_str': ['2023-01-01', '2023-01-02', '2023-01-03']})
df_dates['date'] = pd.to_datetime(df_dates['date_str'])
print("\nDataFrame with datetime column:\n", df_dates)
print("Datetime column type:", df_dates['date'].dtype)

# Series.apply() - to create a new column 'salary_usd' from 'salary'
df_csv['salary_usd'] = df_csv['salary'].apply(lambda x: x / 1.2 if not pd.isna(x) else x)
print("\nDataFrame with 'salary_usd' column (using Series.apply):\n", df_csv)

# DataFrame.apply() - column-wise
# Normalize numerical columns
df_numerical = df_csv[['age', 'salary_usd']].copy()
min_max_scaled = df_numerical.apply(lambda x: (x - x.min()) / (x.max() - x.min()))
print("\nMin-Max Scaled age and salary_usd (using DataFrame.apply):\n", min_max_scaled)

# Series.map() - to categorize cities
city_mapping = {'New York': 'North America', 'London': 'Europe', 'Paris': 'Europe', 'Tokyo': 'Asia', 'Berlin': 'Europe'}
df_csv['continent'] = df_csv['city'].map(city_mapping)
print("\nDataFrame with 'continent' column (using Series.map):\n", df_csv)

# Simple arithmetic
df_csv['bonus'] = df_csv['salary'] * 0.10
print("\nDataFrame with 'bonus' column:\n", df_csv)

# Conditional column creation using np.where
df_csv['seniority'] = np.where(df_csv['age'] >= 35, 'Senior', 'Junior')
print("\nDataFrame with 'seniority' column (using np.where):\n", df_csv)

print("\nOriginal DataFrame for grouping:\n", df_csv)

# Group by 'city' and calculate mean age and salary
grouped_by_city = df_csv.groupby('city').agg({'age': 'mean', 'salary': 'mean'})
print("\nMean age and salary grouped by city:\n", grouped_by_city)

# Group by 'continent' and 'seniority', count employees
grouped_multi = df_csv.groupby(['continent', 'seniority']).size().reset_index(name='count')
print("\nEmployee count by continent and seniority:\n", grouped_multi)

# Using multiple aggregation functions
grouped_stats = df_csv.groupby('city')['salary'].agg(['min', 'max', 'mean', 'std'])
print("\nSalary min, max, mean, std by city:\n", grouped_stats)

df_part1 = pd.DataFrame({'id': [1, 2], 'value': [10, 20]})
df_part2 = pd.DataFrame({'id': [3, 4], 'value': [30, 40]})
print("\nDataFrame Part 1:\n", df_part1)
print("DataFrame Part 2:\n", df_part2)

concatenated_df_rows = pd.concat([df_part1, df_part2], ignore_index=True) # Reset index
print("\nConcatenated (rows):\n", concatenated_df_rows)

df_left = pd.DataFrame({'id': [1, 2], 'A': ['a1', 'a2']})
df_right = pd.DataFrame({'B': ['b1', 'b2'], 'C': ['c1', 'c2']})
print("\nDataFrame Left:\n", df_left)
print("DataFrame Right:\n", df_right)
concatenated_df_cols = pd.concat([df_left, df_right], axis=1)
print("\nConcatenated (columns):\n", concatenated_df_cols)

df_employees = pd.DataFrame({
    'employee_id': [1, 2, 3, 4],
    'name': ['Alice', 'Bob', 'Charlie', 'David'],
    'department_id': [101, 102, 101, 103]
})

df_departments = pd.DataFrame({
    'department_id': [101, 102, 104],
    'department_name': ['HR', 'Engineering', 'Marketing']
})

print("\nEmployees DataFrame:\n", df_employees)
print("\nDepartments DataFrame:\n", df_departments)

# Inner merge (default)
merged_inner = pd.merge(df_employees, df_departments, on='department_id', how='inner')
print("\nInner Merge:\n", merged_inner)

# Left merge
merged_left = pd.merge(df_employees, df_departments, on='department_id', how='left')
print("\nLeft Merge:\n", merged_left)

# Right merge
merged_right = pd.merge(df_employees, df_departments, on='department_id', how='right')
print("\nRight Merge:\n", merged_right)

# Outer merge
merged_outer = pd.merge(df_employees, df_departments, on='department_id', how='outer')
print("\nOuter Merge:\n", merged_outer)

df1_join = pd.DataFrame({'A': ['A0', 'A1'], 'B': ['B0', 'B1']}, index=['K0', 'K1'])
df2_join = pd.DataFrame({'C': ['C0', 'C1'], 'D': ['D0', 'D1']}, index=['K0', 'K2'])

print("\nDataFrame 1 for Join:\n", df1_join)
print("DataFrame 2 for Join:\n", df2_join)

# Left join (default)
joined_df = df1_join.join(df2_join)
print("\nLeft Join (df1.join(df2)):\n", joined_df)

# Outer join
joined_outer_df = df1_join.join(df2_join, how='outer')
print("\nOuter Join:\n", joined_outer_df)

# Create a DataFrame with a DatetimeIndex
dates = pd.date_range(start='2023-01-01', periods=10, freq='D')
ts_data = pd.Series(np.random.randn(10), index=dates)
print("\nTime Series Data:\n", ts_data)

# Resampling (e.g., weekly mean)
weekly_mean = ts_data.resample('W').mean()
print("\nWeekly Mean:\n", weekly_mean)

# Rolling window (e.g., 3-day rolling mean)
rolling_mean = ts_data.rolling(window=3).mean()
print("\n3-day Rolling Mean:\n", rolling_mean)

# Shifting values
shifted_ts = ts_data.shift(1)
print("\nTime Series shifted by 1 day:\n", shifted_ts)

df_fe = pd.DataFrame({
    'user_id': [1, 1, 2, 2, 2, 3],
    'product_id': ['A', 'B', 'A', 'C', 'B', 'A'],
    'price': [10, 20, 10, 30, 25, 10],
    'quantity': [1, 2, 1, 1, 2, 3],
    'timestamp': pd.to_datetime(['2023-01-01 10:00', '2023-01-01 11:00',
                                 '2023-01-02 09:00', '2023-01-02 10:00',
                                 '2023-01-02 12:00', '2023-01-03 14:00'])
})
print("\nOriginal DataFrame for Feature Engineering:\n", df_fe)

# Feature: Total price per transaction
df_fe['total_price'] = df_fe['price'] * df_fe['quantity']
print("\nTotal Price per transaction:\n", df_fe[['user_id', 'product_id', 'total_price']])

# Feature: Number of unique products bought by each user
user_unique_products = df_fe.groupby('user_id')['product_id'].nunique().reset_index(name='num_unique_products')
df_fe = pd.merge(df_fe, user_unique_products, on='user_id', how='left')
print("\nNumber of unique products bought by each user:\n", df_fe[['user_id', 'num_unique_products']].drop_duplicates())


# Feature: Time since last purchase for each user (requires sorting)
df_fe = df_fe.sort_values(by=['user_id', 'timestamp'])
df_fe['time_since_last_purchase'] = df_fe.groupby('user_id')['timestamp'].diff().dt.total_seconds().fillna(0)
print("\nTime since last purchase (seconds):\n", df_fe[['user_id', 'timestamp', 'time_since_last_purchase']])

chat_logs = pd.DataFrame({
    'conversation_id': [1, 1, 1, 2, 2],
    'turn_id': [1, 2, 3, 1, 2],
    'speaker': ['user', 'assistant', 'user', 'user', 'assistant'],
    'text': [
        "What is the capital of France?",
        "The capital of France is Paris.",
        "And Germany?",
        "Tell me a joke.",
        "Why don't scientists trust atoms? Because they make up everything!"
    ]
})
print("\nOriginal Chat Logs:\n", chat_logs)

# Goal: Transform into (instruction, response) pairs for LLM fine-tuning
# A simple approach for alternating user/assistant turns

def create_instruction_response(group):
    instructions = []
    responses = []
    current_instruction = []

    for _, row in group.iterrows():
        if row['speaker'] == 'user':
            current_instruction.append(row['text'])
        elif row['speaker'] == 'assistant' and current_instruction:
            instructions.append("\n".join(current_instruction))
            responses.append(row['text'])
            current_instruction = [] # Reset for next instruction

    return pd.DataFrame({'instruction': instructions, 'response': responses})

llm_finetune_df = chat_logs.groupby('conversation_id').apply(create_instruction_response).reset_index(drop=True)
print("\nData for LLM Fine-tuning (Instruction-Response Pairs):\n", llm_finetune_df)

# Further processing: Adding a 'system' prompt
llm_finetune_df['full_prompt'] = "### Instruction:\n" + llm_finetune_df['instruction'] + \
                                 "\n\n### Response:\n" + llm_finetune_df['response']
print("\nFull Prompt for LLM Fine-tuning:\n", llm_finetune_df['full_prompt'].iloc[0])

agent_performance_data = pd.DataFrame({
    'episode_id': [1, 1, 2, 2, 2, 3],
    'step': [1, 2, 1, 2, 3, 1],
    'action': ['move_left', 'collect_item', 'move_right', 'attack', 'move_left', 'idle'],
    'reward': [0, 10, 0, -5, 0, 0],
    'status': ['ongoing', 'completed', 'ongoing', 'ongoing', 'failed', 'completed']
})
print("\nAgent Performance Data:\n", agent_performance_data)

# Calculate total reward per episode
episode_rewards = agent_performance_data.groupby('episode_id')['reward'].sum().reset_index(name='total_reward')
print("\nTotal Reward per Episode:\n", episode_rewards)

# Find episodes that ended in 'failed' status
failed_episodes = agent_performance_data[agent_performance_data['status'] == 'failed']['episode_id'].unique()
print("\nFailed Episodes:", failed_episodes)

# Analyze action distribution
action_distribution = agent_performance_data['action'].value_counts(normalize=True).reset_index(name='proportion')
print("\nAction Distribution:\n", action_distribution)

# Illustrating vectorization vs. loop
large_df = pd.DataFrame(np.random.randint(0, 100, size=(1_000_000, 2)), columns=['A', 'B'])

start_time = time.time()
large_df['C_vectorized'] = large_df['A'] + large_df['B']
print(f"\nVectorized operation took: {time.time() - start_time:.6f} seconds")

start_time = time.time()
large_df['C_apply'] = large_df.apply(lambda row: row['A'] + row['B'], axis=1)
print(f"Apply operation took: {time.time() - start_time:.6f} seconds")

# Don't even try iterrows for 1M rows if you can avoid it!
# start_time = time.time()
# results = []
# for index, row in large_df.iterrows():
#     results.append(row['A'] + row['B'])
# large_df['C_loop'] = results
# print(f"Loop operation took: {time.time() - start_time:.6f} seconds")

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns # Import seaborn as well, we'll use it for data later

# Set a style for better aesthetics (optional)
plt.style.use('ggplot')

# Simple line plot
x = np.linspace(0, 10, 100)
y = np.sin(x)

plt.figure(figsize=(8, 4)) # Create a figure and specify its size
plt.plot(x, y)
plt.title("Simple Sine Wave")
plt.xlabel("X-axis Label")
plt.ylabel("Y-axis Label")
plt.grid(True)
plt.show() # Display the plot

# Create a figure and a set of subplots
fig, ax = plt.subplots(figsize=(10, 5)) # fig is the Figure, ax is the Axes object

ax.plot(x, np.cos(x), label='Cosine')
ax.plot(x, np.sin(x), label='Sine')
ax.set_title("Sine and Cosine Waves")
ax.set_xlabel("Angle (radians)")
ax.set_ylabel("Amplitude")
ax.legend() # Show the legend
ax.grid(True)
plt.show()

fig, axes = plt.subplots(2, 2, figsize=(12, 8)) # 2 rows, 2 columns of subplots

# Plot 1: Top-left
axes[0, 0].plot(x, y, color='red')
axes[0, 0].set_title("Sine")

# Plot 2: Top-right
axes[0, 1].plot(x, y**2, color='blue')
axes[0, 1].set_title("Sine Squared")

# Plot 3: Bottom-left
axes[1, 0].scatter(x, y, color='green', marker='o', s=10) # s is marker size
axes[1, 0].set_title("Sine Scatter")

# Plot 4: Bottom-right
axes[1, 1].hist(np.random.randn(1000), bins=30, color='purple', alpha=0.7)
axes[1, 1].set_title("Histogram")

plt.tight_layout() # Adjust subplot parameters for a tight layout
plt.show()

plt.figure(figsize=(8, 5))
plt.plot(x, y, color='steelblue', linestyle='--', linewidth=2, marker='o', markersize=5, label='Sine Wave')
plt.title("Customized Sine Wave", fontsize=16)
plt.xlabel("X-axis", fontsize=12)
plt.ylabel("Y-axis", fontsize=12)
plt.legend(loc='upper right')
plt.ylim(-1.5, 1.5) # Set y-axis limits
plt.text(7, 1.2, "Peak Value!", fontsize=10, color='darkgreen')
plt.annotate('Local Max', xy=(np.pi/2, 1), xytext=(np.pi/2 + 1, 1.3),
             arrowprops=dict(facecolor='black', shrink=0.05),
             fontsize=10)
plt.grid(True, linestyle=':', alpha=0.6)
plt.savefig('custom_sine_wave.png', dpi=300) # Save plot with high resolution
plt.show()

# Generate some dummy data
categories = ['A', 'B', 'C', 'D', 'E']
values = np.random.randint(10, 100, size=len(categories))
data_for_box = [np.random.normal(0, 1, 100), np.random.normal(1, 1.5, 100), np.random.normal(-1, 0.5, 100)]

fig, axes = plt.subplots(1, 2, figsize=(14, 6))

# Bar Chart
axes[0].bar(categories, values, color=['skyblue', 'lightcoral', 'lightgreen', 'gold', 'plum'])
axes[0].set_title("Bar Chart of Categories")
axes[0].set_xlabel("Category")
axes[0].set_ylabel("Value")

# Box Plot
axes[1].boxplot(data_for_box, labels=['Group 1', 'Group 2', 'Group 3'])
axes[1].set_title("Box Plot of Distributions")
axes[1].set_ylabel("Value")

plt.tight_layout()
plt.show()

# Load a built-in Seaborn dataset (often used for examples)
tips_df = sns.load_dataset('tips')
print("Tips DataFrame (first 5 rows):\n", tips_df.head())

plt.figure(figsize=(8, 6))
sns.histplot(data=tips_df, x='total_bill', kde=True) # kde=True adds a Kernel Density Estimate
plt.title("Distribution of Total Bill")
plt.xlabel("Total Bill Amount ($)")
plt.ylabel("Count")
plt.show()

fig, axes = plt.subplots(1, 2, figsize=(12, 5))

# Histogram with KDE
sns.histplot(tips_df['total_bill'], kde=True, ax=axes[0])
axes[0].set_title("Total Bill Distribution (Histplot)")

# KDE Plot
sns.kdeplot(tips_df['total_bill'], fill=True, ax=axes[1])
axes[1].set_title("Total Bill Distribution (KDE Plot)")

plt.tight_layout()
plt.show()

# Scatter plot with regression line
plt.figure(figsize=(8, 6))
sns.regplot(data=tips_df, x='total_bill', y='tip')
plt.title("Total Bill vs. Tip with Regression Line")
plt.show()

# Pair Plot (shows relationships between all numerical variables)
# Often used for initial EDA
# sns.pairplot(tips_df, hue='sex') # Hue adds color based on a categorical variable
# plt.suptitle("Pair Plot of Tips Dataset", y=1.02)
# plt.show()

fig, axes = plt.subplots(1, 2, figsize=(14, 6))

# Box plot
sns.boxplot(data=tips_df, x='day', y='total_bill', ax=axes[0])
axes[0].set_title("Total Bill by Day (Boxplot)")

# Violin plot (combines box plot with kernel density estimate)
sns.violinplot(data=tips_df, x='day', y='total_bill', hue='sex', split=True, ax=axes[1])
axes[1].set_title("Total Bill by Day and Sex (Violinplot)")
axes[1].legend(title='Sex')

plt.tight_layout()
plt.show()