# učitavanje biblioteka
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

# podešavanje colab+drive
from google.colab import drive
drive.mount('/content/drive')

# podešavanja ispisa
pd.set_option('display.float_format', lambda x: '%.2f' % x) # prikaz df na 2 decimale (npr. za describe)

Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).

!gdown 1mig6WygA9fCZlKSRf0FQFLqXc241-qVd

Downloading...
From (original): https://drive.google.com/uc?id=1mig6WygA9fCZlKSRf0FQFLqXc241-qVd
From (redirected): https://drive.google.com/uc?id=1mig6WygA9fCZlKSRf0FQFLqXc241-qVd&confirm=t&uuid=c0c142a4-9d56-478e-93c9-ceb1a68ca2c1
To: /content/city_temperature.csv
100% 141M/141M [00:01<00:00, 73.6MB/s]

# učitavanje u dataframe format
df=pd.read_csv("city_temperature.csv")
print(df.shape)
print(df.dtypes)
df.head()

(2906327, 8)
Region             object
Country            object
State              object
City               object
Month               int64
Day                 int64
Year                int64
AvgTemperature    float64
dtype: object

/tmp/ipython-input-22-3136775249.py:2: DtypeWarning: Columns (2) have mixed types. Specify dtype option on import or set low_memory=False.
  df=pd.read_csv("city_temperature.csv")

# format podataka

# Pretvaranje farenhajta u celzijuse
df['AvgTemperatureC'] = (df['AvgTemperature']-32) * (5/9)
df.head()

# df.isnull() ## changes all values to True/False
print(df.isnull().sum()) ## sums by columns, e.g. 1st columns has 32 nulls while the 2nd column has 5
# df.loc[:].isna()
print(df.shape)
print(df.dropna().shape)

Region                   0
Country                  0
State              1450990
City                     0
Month                    0
Day                      0
Year                     0
AvgTemperature           0
AvgTemperatureC          0
dtype: int64
(2906327, 9)
(1455337, 9)

unique_regions = df['Region'].dropna().unique()
unique_countries = df['Country'].dropna().unique()
unique_states = df['State'].dropna().unique()
print(unique_regions)
print(unique_countries)
print(unique_states)

['Africa' 'Asia' 'Australia/South Pacific' 'Europe' 'Middle East'
 'North America' 'South/Central America & Carribean']
['Algeria' 'Burundi' 'Benin' 'Central African Republic' 'Congo' 'Egypt'
 'Ethiopia' 'Gabon' 'Gambia' 'Guinea' 'Guinea-Bissau' 'Ivory Coast'
 'Kenya' 'Morocco' 'Madagascar' 'Mauritania' 'Malawi' 'Mozambique'
 'Namibia' 'Nigeria' 'Senegal' 'Sierra Leone' 'South Africa' 'Togo'
 'Tunisia' 'Tanzania' 'Uganda' 'Zambia' 'Bangladesh' 'China' 'Hong Kong'
 'India' 'Indonesia' 'Japan' 'Kazakhstan' 'Kyrgyzstan' 'Laos' 'Malaysia'
 'Mongolia' 'Myanmar (Burma)' 'Nepal' 'North Korea' 'Pakistan'
 'Philippines' 'Singapore' 'South Korea' 'Sri Lanka' 'Taiwan' 'Tajikistan'
 'Thailand' 'Turkmenistan' 'Uzbekistan' 'Vietnam' 'Australia'
 'New Zealand' 'Albania' 'Austria' 'Belarus' 'Belgium' 'Bulgaria'
 'Croatia' 'Cyprus' 'Czech Republic' 'Denmark' 'Finland' 'France'
 'Germany' 'Georgia' 'Greece' 'Hungary' 'Iceland' 'Ireland' 'Italy'
 'Latvia' 'Macedonia' 'The Netherlands' 'Norway' 'Poland' 'Portugal'
 'Romania' 'Russia' 'Serbia-Montenegro' 'Slovakia' 'Spain' 'Sweden'
 'Switzerland' 'Ukraine' 'United Kingdom' 'Yugoslavia' 'Bahrain' 'Israel'
 'Jordan' 'Kuwait' 'Lebanon' 'Oman' 'Qatar' 'Saudi Arabia' 'Syria'
 'Turkey' 'United Arab Emirates' 'Canada' 'Mexico' 'Argentina' 'Bahamas'
 'Bermuda' 'Belize' 'Bolivia' 'Barbados' 'Brazil' 'Colombia' 'Costa Rica'
 'Cuba' 'Dominican Republic' 'Equador' 'Guatemala' 'Guyana' 'Haiti'
 'Honduras' 'Nicaragua' 'Panama' 'Peru' 'Suriname' 'Uruguay' 'Venezuela'
 'US']
['Alabama' 'Alaska' 'Arizona' 'Arkansas' 'California' 'Colorado'
 'Connecticut' 'Delaware' 'District of Columbia' 'Maryland' 'Florida'
 'Georgia' 'Hawaii' 'Idaho' 'Illinois' 'Indiana' 'Iowa' 'Kansas'
 'Kentucky' 'Louisiana' 'Maine' 'Massachusetts' 'Michigan' 'Minnesota'
 'Mississippi' 'Missouri' 'Montana' 'Nebraska' 'Nevada' 'New Hampshire'
 'New Jersey' 'New Mexico' 'New York' 'North Carolina' 'North Dakota'
 'Ohio' 'Oklahoma' 'Oregon' 'Pennsylvania' 'Rhode Island' 'South Carolina'
 'South Dakota' 'Tennessee' 'Texas' 'Utah' 'Vermont' 'Virginia'
 'Washington' 'West Virginia' 'Wisconsin' 'Wyoming'
 'Additional Territories']

# drop(ind,axis,inplace)
# df.drop([1],axis=0) ## 1 = column axis, 0 = row axis, inplace=True
df.drop(columns=['State'],inplace=True)

print(df.isna().sum()) ## da rijeseno je

print(df.describe())

Region             0
Country            0
City               0
Month              0
Day                0
Year               0
AvgTemperature     0
AvgTemperatureC    0
dtype: int64
           Month        Day       Year  AvgTemperature  AvgTemperatureC
count 2906327.00 2906327.00 2906327.00      2906327.00       2906327.00
mean        6.47      15.72    2006.62           56.00            13.34
std         3.46       8.80      23.38           32.12            17.85
min         1.00       0.00     200.00          -99.00           -72.78
25%         3.00       8.00    2001.00           45.80             7.67
50%         6.00      16.00    2007.00           62.50            16.94
75%         9.00      23.00    2013.00           75.50            24.17
max        12.00      31.00    2020.00          110.00            43.33

# proveriti koje godine se javljaju u bazi
print(df['Year'].unique())   ## year 201 and 200 don't seem right

[1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008
 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020  201  200]

df['Year'] = df['Year'].replace(200,np.nan)
df['Year'] = df['Year'].replace(201,np.nan)
df['Day'] = df['Day'].replace(0,np.nan)

print(df.isna().sum())
print(df.isna().sum() / df.shape[0] *100 )
# print(df.shape[0])

Region               0
Country              0
City                 0
Month                0
Day                  8
Year               440
AvgTemperature       0
AvgTemperatureC      0
dtype: int64
Region            0.00
Country           0.00
City              0.00
Month             0.00
Day               0.00
Year              0.02
AvgTemperature    0.00
AvgTemperatureC   0.00
dtype: float64

# brisanje, funkcija dropna(inplace, axis)
df.dropna(inplace = True, axis=0)

# zamena medijanom bi se radila na sledeći način:
#df['Day'].fillna(df['Day'].median(), inplace = True)
#df['Year'].fillna(df['Year'].median(), inplace = True)

## isto dodje dal ides 60C ili 60F posto gledamo gradove. Ne znam na koju skalu misle
print(df.loc[df['AvgTemperature'] < -60,'AvgTemperature'] )
# print(df[df['AvgTemperatureC'] < -60] )

df.loc[df['AvgTemperature'] < -60,'AvgTemperature'] = np.nan
df.loc[df['AvgTemperatureC'] < -60,'AvgTemperatureC'] = np.nan
print('AvgTemperature: broj uzoraka:', df['AvgTemperature'].isna().sum(), 'tj.', df['AvgTemperature'].isna().sum()/len(df)*100, '%')
print('AvgTemperature: broj uzoraka:', df['AvgTemperatureC'].isna().sum(), 'tj.', df['AvgTemperatureC'].isna().sum()/len(df)*100, '%')
# print(df[df['AvgTemperature'] == np.nan] )

220       -99.00
221       -99.00
408       -99.00
409       -99.00
1453      -99.00
           ...  
2905471   -99.00
2905534   -99.00
2905689   -99.00
2905726   -99.00
2905749   -99.00
Name: AvgTemperature, Length: 79224, dtype: float64
AvgTemperature: broj uzoraka: 79224 tj. 2.726335129576971 %
AvgTemperature: broj uzoraka: 79224 tj. 2.726335129576971 %

# prebrojati validne vrednosti po obeležju za svaki grad (samo za temperaturu postoje nevalidne vrednosti, tj. NaN)
# koristiti groupby i count. Dobijen je DataFrame čija su imena vrsta ustvari imena gradova.
# prolazeći for petljom kroz gradove proveriti za svaki grad da li ima preko 10% nevalidnih vrednosti u koloni AvgTemperature
# gradove koji imaju preko 10% nevaidnih vrednosti, smestiti u listu gradova za brisanje. Ispisati listu

# print(df.columns)
# print(df.shape)
# print(df.groupby(['City']).count()) ## count() ignores NaN

# city = df.groupby(['City'], dropna=False) ## keep the Nan values

del_city = []
for city in df.groupby(['City'], dropna=False):
  ## if you put df.groupby(['City']) then you'll get city[0] = ('Chicago',)
  ## if you put df.groupby('City') then you'll get city[0] = 'Chicago'
  # print('city name=', city[0])

  # print('city values=', city[1])  ## grouped data for a city
  # print('city temps=', city[1]['AvgTemperature'])   ## the temps of a city
  # print('city temps=', city[1]['AvgTemperature'].isna().sum() , ' ?? ', city[1]['AvgTemperature'].count())    ## is null compared to not null

  if  city[1]['AvgTemperature'].count() / city[1]['AvgTemperature'].shape[0]  < 0.9 :
    ## if you put df.groupby(['City']) then you'll get city[0] = ('Chicago',)
    ## if you put df.groupby('City') then you'll get city[0] = 'Chicago'
    city_name = city[0] if isinstance(city[0],str) else city[0][0]
    del_city.append(city_name)


# print(len(del_city))
print(del_city)
old_del_city = del_city


gb = df.groupby(by='City').count() # broji validne vrednosti po obeležju za svaki grad (samo za AvgTemperature postoje NaN)
del_city = []
for i in df['City'].unique():
  # print(i,"=>",gb.loc[i, 'AvgTemperature'])
  # print(i,"=>",gb.loc[i, 'Day'])
    if gb.loc[i, 'AvgTemperature']/gb.loc[i,'Day'] < 0.9:
        del_city.append(i)

print(del_city)
print(set(del_city).intersection(old_del_city))

if len([ a for a in old_del_city if a in del_city]) == len(del_city):
  print("YES")

['Addis Ababa', 'Banjul', 'Bissau', 'Bonn', 'Bujumbura', 'Conakry', 'Dhaka', 'Dusanbe', 'Frankfurt', 'Freetown', 'Georgetown', 'Guadalajara', 'Hamburg', 'Jakarta', 'Kampala', 'Katmandu', 'Lagos', 'Lilongwe', 'Muscat', 'Nairobi', 'Nicosia', 'Paramaribo', 'Port au Prince', 'San Juan Puerto Rico', 'Tirana']
['Bujumbura', 'Addis Ababa', 'Banjul', 'Conakry', 'Bissau', 'Nairobi', 'Lilongwe', 'Lagos', 'Freetown', 'Kampala', 'Dhaka', 'Jakarta', 'Katmandu', 'Dusanbe', 'Tirana', 'Nicosia', 'Bonn', 'Frankfurt', 'Hamburg', 'Muscat', 'Guadalajara', 'Georgetown', 'Port au Prince', 'Paramaribo', 'San Juan Puerto Rico']
{'Nairobi', 'Guadalajara', 'Katmandu', 'Dhaka', 'Dusanbe', 'Bujumbura', 'Lilongwe', 'Paramaribo', 'Freetown', 'Georgetown', 'Nicosia', 'Banjul', 'Frankfurt', 'Addis Ababa', 'Muscat', 'Bissau', 'San Juan Puerto Rico', 'Bonn', 'Lagos', 'Hamburg', 'Conakry', 'Kampala', 'Tirana', 'Port au Prince', 'Jakarta'}
YES

print('Broj uzoraka i obelezja pre brisanja: ', df.shape)
print('Broj jedinstvenih gradova pre brisanja: ', len(df['City'].unique()))
df = df[~df['City'].isin(del_city)]
#for c in del_city:
#    df.drop(df.loc[df['City']==c].index, axis=0, inplace=True)
print('Broj uzoraka i obelezja posle brisanja: ', df.shape)
print('Broj jedinstvenih gradova posle brisanja: ', len(df['City'].unique()))

Broj uzoraka i obelezja pre brisanja:  (2905879, 8)
Broj jedinstvenih gradova pre brisanja:  321
Broj uzoraka i obelezja posle brisanja:  (2716826, 8)
Broj jedinstvenih gradova posle brisanja:  296

print(df.isna().sum())

Region                 0
Country                0
City                   0
Month                  0
Day                    0
Year                   0
AvgTemperature     22884
AvgTemperatureC    22884
dtype: int64

## median ili mean
# print(df['AvgTemperature'].fillna(df['AvgTemperature'].median())) ## median ili mean je ok

## ffill => forward fill => ovako proslijedjujem zadnju validnu vrijednost unaprijed
print(df['AvgTemperature'].fillna(method='ffill')) ## median ili mean je ok
print(df['AvgTemperature'].fillna(method='ffill').isna().sum()) ## median ili mean je ok

## bfill => bacward fill => ovako proslijedjujem zadnju validnu vrijednost unaprijed ali ideci od zada dataframe-a
print(df['AvgTemperature'].fillna(method='bfill')) ## median ili mean je ok
print(df['AvgTemperature'].fillna(method='bfill').isna().sum()) ## median ili mean je ok

df['AvgTemperature'] = df['AvgTemperature'].fillna(method='ffill')

0         64.20
1         49.40
2         48.80
3         46.40
4         47.90
           ... 
2899535   42.30
2899536   43.10
2899537   37.80
2899538   41.90
2899539   48.50
Name: AvgTemperature, Length: 2716826, dtype: float64
0
0         64.20
1         49.40
2         48.80
3         46.40
4         47.90
           ... 
2899535   42.30
2899536   43.10
2899537   37.80
2899538   41.90
2899539   48.50
Name: AvgTemperature, Length: 2716826, dtype: float64
0

/tmp/ipython-input-37-4213398936.py:5: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.
  print(df['AvgTemperature'].fillna(method='ffill')) ## median ili mean je ok
/tmp/ipython-input-37-4213398936.py:6: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.
  print(df['AvgTemperature'].fillna(method='ffill').isna().sum()) ## median ili mean je ok
/tmp/ipython-input-37-4213398936.py:9: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.
  print(df['AvgTemperature'].fillna(method='bfill')) ## median ili mean je ok
/tmp/ipython-input-37-4213398936.py:10: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.
  print(df['AvgTemperature'].fillna(method='bfill').isna().sum()) ## median ili mean je ok
/tmp/ipython-input-37-4213398936.py:12: FutureWarning: Series.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.
  df['AvgTemperature'] = df['AvgTemperature'].fillna(method='ffill')

# postaviti region za indeks zbog lakšeg pretraživanja pomoću dataframe.loc[]
df_region = df.set_index('Region')
df_region.head()

for region in unique_regions:
  plt.hist(df_region.loc[region,'AvgTemperature'],alpha=0.5, label=region)

plt.xlabel('temperature')
plt.legend()
plt.show()


# for region in unique_regions:
  # for year in df_region.loc[region,['AvgTemperature','Year']].groupby('Year'):
  #   print(year)

# boxplot
     #boxplot za Evropu za AvgTemperature obeležje
plt.boxplot(df_region.loc['Europe', 'AvgTemperature'])
plt.ylabel('Prosečna temperatura (℃)')
plt.xlabel('Region')
plt.xticks([1], ["Evropa"])
plt.grid()
plt.show()

# prikazati uporedo Evropu i Severnu Ameriku. Dodati nazive osa, mrežu i imena boxplot-ova
plt.boxplot(
    [df_region.loc['Europe', 'AvgTemperature'],df_region.loc['North America', 'AvgTemperature']]
    ,labels=['Europe','North America']
  )
plt.ylabel('Prosečna temperatura (℃)')
plt.xlabel('Region')
plt.grid(visible=True,axis='both')
plt.xticks([1,2], ["Evropa",'North America'])
# plt.grid()
plt.show()

/tmp/ipython-input-40-3895612982.py:11: MatplotlibDeprecationWarning: The 'labels' parameter of boxplot() has been renamed 'tick_labels' since Matplotlib 3.9; support for the old name will be dropped in 3.11.
  plt.boxplot(

# histogram
plt.hist(df_region.loc['Europe','AvgTemperature'],alpha=0.5, label=['Europe'],bins=50)
plt.hist(df_region.loc['North America','AvgTemperature'],alpha=0.5, label=['NA'],bins=50)

plt.xlabel('temperature')
plt.legend()
plt.show()

# indeksirati po godini
df_year = df.set_index('Year')
print(df_year)

                Region  Country      City  Month   Day  AvgTemperature  \
Year                                                                     
1995.00         Africa  Algeria   Algiers      1  1.00           64.20   
1995.00         Africa  Algeria   Algiers      1  2.00           49.40   
1995.00         Africa  Algeria   Algiers      1  3.00           48.80   
1995.00         Africa  Algeria   Algiers      1  4.00           46.40   
1995.00         Africa  Algeria   Algiers      1  5.00           47.90   
...                ...      ...       ...    ...   ...             ...   
2020.00  North America       US  Cheyenne      5  9.00           42.30   
2020.00  North America       US  Cheyenne      5 10.00           43.10   
2020.00  North America       US  Cheyenne      5 11.00           37.80   
2020.00  North America       US  Cheyenne      5 12.00           41.90   
2020.00  North America       US  Cheyenne      5 13.00           48.50   

         AvgTemperatureC  
Year                      
1995.00            17.89  
1995.00             9.67  
1995.00             9.33  
1995.00             8.00  
1995.00             8.83  
...                  ...  
2020.00             5.72  
2020.00             6.17  
2020.00             3.22  
2020.00             5.50  
2020.00             9.17  

[2716826 rows x 7 columns]

# histogram
# df_year.loc[1995,'AvgTemperature']
plt.hist(df_year.loc[1995,'AvgTemperature'],label=['1995'],alpha=0.5,bins=50)
plt.hist(df_year.loc[2005,'AvgTemperature'],label=['1995'],alpha=0.5,bins=50)
plt.show()

from scipy.stats import kurtosis
from scipy.stats import skew

print('koef.asimetrije:  %.2f' % skew(df_region.loc['Europe','AvgTemperature']))
print('koef.spljoštenosti:  %.2f' % kurtosis(df_region.loc['Europe','AvgTemperature']))
##

print('skew:',skew(np.arange(0,10)))
print('kurtosis:',kurtosis(np.arange(0,10)))
print('skew:',skew([1+i*0.1 for i in range(0,10)]))
print('kurtosis:',kurtosis([1+i*0.1 for i in range(0,10)]))

koef.asimetrije:  -0.24
koef.spljoštenosti:  -0.23
skew: 0.0
kurtosis: -1.2242424242424244
skew: -1.7569547428617478e-15
kurtosis: -1.224242424242425

# histogram sa obeleženim osama
import seaborn as sb
from scipy.stats import norm
temp_europe = df_region.loc['Europe','AvgTemperature']
sb.distplot(temp_europe, fit=norm)
# sb.displot(temp_europe,kind='hist') ## the new displot does not give the same result
# sb.displot(temp_europe,kind='kde')
plt.xlabel('Temperatura (℃)')
plt.ylabel('Verovatnoća')

/tmp/ipython-input-45-523256125.py:5: UserWarning: 

`distplot` is a deprecated function and will be removed in seaborn v0.14.0.

Please adapt your code to use either `displot` (a figure-level function with
similar flexibility) or `histplot` (an axes-level function for histograms).

For a guide to updating your code to use the new functions, please see
https://gist.github.com/mwaskom/de44147ed2974457ad6372750bbe5751

  sb.distplot(temp_europe, fit=norm)

gb = df.groupby(['Region','Month']).mean(numeric_only=True)
gb

for region in df['Region'].unique():
  print(region)
  print(gb.loc[region,'AvgTemperature'])

# DataFrame gb ima imena vrsta koje odgovaraju imenima Regiona,
# a Prosečne temperature po mesecima nalaze se u koloni AvgTemperature
# na x-osu grafika staviti vrednosto od 1 do 12 koje će označavati mesec
# dodeliti imena osama i prikazati legendu grafika


# print(gb.loc['Europe','AvgTemperature'])
# print(gb.loc['Asia','AvgTemperature'])

plt.plot(gb.loc['Europe','AvgTemperature'])
plt.plot(gb.loc['Asia','AvgTemperature'])
plt.show()

plt.scatter(y=gb.loc['Europe','AvgTemperature'],x=gb.loc['Europe','AvgTemperature'].index)
plt.scatter(y=gb.loc['Asia','AvgTemperature'],x=gb.loc['Asia','AvgTemperature'].index)
plt.show()

# Nacrtati iste podatke na grafiku rasipanja (zavisnost prosečne temperature u Aziji od prosečne temperature u Evropi)

            # grafik rasipanja
plt.scatter(x=gb.loc['Europe','AvgTemperatureC'],y=gb.loc['Asia','AvgTemperatureC'])
plt.xlabel('Mesečna prosečna temperatura u Evropi (℃)');
plt.ylabel('Mesečna prosečna temperatura u Aziji (℃)');
plt.show()

df_month = pd.DataFrame()
for i in df_region.index.unique():
    df_month[i] = gb.loc[i, 'AvgTemperature'] ## df_month[i] = 12 rows of AvgTemp for each month
c=df_month['Europe'].corr(df_month['Asia'])
print("korelacija: %.3f" % c)

matrica_korelacije = df_month.corr()
# print(df_month.corr())
df_month.corr()['Europe']
# df_month.corr()

import seaborn as sb

sb.heatmap(matrica_korelacije, annot=True)

!jupyter nbconvert --to html "/content/drive/MyDrive/Colab Notebooks/zadatak1_EDA_moje_rjesenje.ipynb"

	Region	Country	State	City	Month	Day	Year	AvgTemperature
0	Africa	Algeria	NaN	Algiers	1	1	1995	64.20
1	Africa	Algeria	NaN	Algiers	1	2	1995	49.40
2	Africa	Algeria	NaN	Algiers	1	3	1995	48.80
3	Africa	Algeria	NaN	Algiers	1	4	1995	46.40
4	Africa	Algeria	NaN	Algiers	1	5	1995	47.90

	Country	City	Month	Day	Year	AvgTemperature	AvgTemperatureC
Region
Africa	Algeria	Algiers	1	1.00	1995.00	64.20	17.89
Africa	Algeria	Algiers	1	2.00	1995.00	49.40	9.67
Africa	Algeria	Algiers	1	3.00	1995.00	48.80	9.33
Africa	Algeria	Algiers	1	4.00	1995.00	46.40	8.00
Africa	Algeria	Algiers	1	5.00	1995.00	47.90	8.83

Vežba 3: EDA¶