
    i4#                        d dl Z d dlmZ d dlmc mZ d dlmZ d dlZd dl	m
Z
 d dlmZ d dlZ G d dej                        Z G d dej                        Z G d dej                        Zdd	Zdd
Zd Zedk(  r e        yy)    N)tqdmc                   *     e Zd ZdZd fd	Zd Z xZS )HyperGasLinearzLinear layer where weights and biases are Gaussian distributions.
    Uses reparameterization trick: w = mu + sigma * epsilon, epsilon ~ N(0,1).
    c                 4   t         |           || _        || _        t	        j
                  t        j                  ||      j                  dd            | _	        t	        j
                  t        j                  ||      j                  dd            | _
        t	        j
                  t        j                  |      j                               | _        t	        j
                  t        j                  |      j                  dd            | _        || _        y )Nr   {Gz?皙?)super__init__in_featuresout_featuresnn	ParametertorchTensornormal_w_muw_log_sigmazero_b_mub_log_sigmaprior_sigma)selfr   r   r   	__class__s       7/home/per/Documents/Phase change AI/cifar10_hypergas.pyr   zHyperGasLinear.__init__   s    &( LLlK!H!P!PQRTX!YZ	<<\;(O(W(WXZ\_(`aLLl!;!A!A!CD	<<\(B(J(J2s(ST '    c                 |   t        j                  | j                        }t        j                  | j                        }t        j                  | j
                        }t        j                  | j                        }| j                  ||z  z   }| j                  ||z  z   }t        j                  | j
                  t        j                  t        j                  | j                              z
  |dz  | j                  dz  z   d| j                  dz  z  z  z   dz
        }t        j                  | j                  t        j                  t        j                  | j                              z
  |dz  | j                  dz  z   d| j                  dz  z  z  z   dz
        }	||	z   | _        t        j                  |||      S )N         ?)r   
randn_liker   r   expr   r   sumlogtensorr   klFlinear
r   xw_epsb_epsw_sigmab_sigmawbkl_wkl_bs
             r   forwardzHyperGasLinear.forward   sn     +  +))D,,-))D,,-II%'II%'
 yy))EIIell4CSCS6T,UU!1*tyy!|3D<L<La<O8OPQSVW Xyy))EIIell4CSCS6T,UU!1*tyy!|3D<L<La<O8OPQSVW X+xx1a  r   r	   __name__
__module____qualname____doc__r   r2   __classcell__r   s   @r   r   r      s    '!r   r   c                   *     e Zd ZdZd fd	Zd Z xZS )HyperGasConv2dz82D convolutional layer with Gaussian weights and biases.c                    t         |           || _        || _        t	        |t
              r||fn|| _        || _        || _        || j                  d   z  | j                  d   z  }t        j                  t        j                  ||g| j                   j                  dd            | _        t        j                  t        j                  ||g| j                   j                  dd            | _        t        j                  t        j                  |      j!                               | _        t        j                  t        j                  |      j                  dd            | _        || _        y )Nr      r   r   r	   )r
   r   in_channelsout_channels
isinstanceintkernel_sizestridepaddingr   r   r   r   r   r   r   r   r   r   r   )	r   r?   r@   rC   rD   rE   r   fan_inr   s	           r   r   zHyperGasConv2d.__init__6   s(   &(9CKQT9UK5[ft//22T5E5Ea5HHLLlK![$JZJZ![!c!cdegk!lm	<<\;(bQUQaQa(b(j(jkmor(stLLl!;!A!A!CD	<<\(B(J(J2s(ST&r   c                    t        j                  | j                        }t        j                  | j                        }t        j                  | j
                        }t        j                  | j                        }| j                  ||z  z   }| j                  ||z  z   }t        j                  | j
                  t        j                  t        j                  | j                              z
  |dz  | j                  dz  z   d| j                  dz  z  z  z   dz
        }t        j                  | j                  t        j                  t        j                  | j                              z
  |dz  | j                  dz  z   d| j                  dz  z  z  z   dz
        }	||	z   | _        t        j                  |||| j                  | j                        S )Nr   r   rD   rE   )r   r    r   r   r!   r   r   r"   r#   r$   r   r%   r&   conv2drD   rE   r(   s
             r   r2   zHyperGasConv2d.forwardE   sx     +  +))D,,-))D,,-II%'II%'yy))EIIell4CSCS6T,UU!1*tyy!|3D<L<La<O8OPQSVW Xyy))EIIell4CSCS6T,UU!1*tyy!|3D<L<La<O8OPQSVW X+xx1aT\\JJr   )r>   r   r	   r4   r:   s   @r   r<   r<   4   s    B'Kr   r<   c                   2     e Zd ZdZd fd	ZddZd Z xZS )HyperGasNetz"Bayesian CNN for CIFAR-10 (32x32).c                     t         |           t        dddd|      | _        t        dddd|      | _        t        j                  dd      | _        t        dd|	      | _	        t        dd
|	      | _
        y )N       r>   )rE   r   @   r   i      r   
   )r
   r   r<   conv1conv2r   	MaxPool2dpoolr   fc1fc2)r   r   r   s     r   r   zHyperGasNet.__init__X   sh    #Ar1a[Q
#BAqkR
LLA&	!*c{K!#r{Cr   c           	         |st        j                  t        j                  || j                  j                  | j                  j
                  dd            }| j                  |      }t        j                  t        j                  || j                  j                  | j                  j
                  dd            }| j                  |      }|j                  |j                  d      d      }t        j                  t        j                  || j                  j                  | j                  j
                              }t        j                  || j                  j                  | j                  j
                        }|S t        j                  | j                  |            }| j                  |      }t        j                  | j                  |            }| j                  |      }|j                  |j                  d      d      }t        j                  | j                  |            }| j                  |      }|S )Nr>   rH   r   )r&   relurI   rS   r   r   rV   rT   viewsizer'   rW   rX   )r   r)   samples      r   r2   zHyperGasNet.forward`   sq   qxx4::??DJJOOAWXYZA		!Aqxx4::??DJJOOAWXYZA		!Aqvvay"%Aqxx488==$((--@AADHHMM488==9AH tzz!}%A		!Atzz!}%A		!Aqvvay"%Atxx{#AAHr   c                     | j                   j                  | j                  j                  z   | j                  j                  z   | j                  j                  z   dz  }|S )z&Sum of KL divergences from all layers.i`  )rS   r%   rT   rW   rX   )r   r%   s     r   kl_losszHyperGasNet.kl_lossw   s@    jjmmdjjmm+dhhkk9DHHKKG5P	r   r3   )T)r5   r6   r7   r8   r   r2   r`   r9   r:   s   @r   rK   rK   V   s    ,D.r   rK   c           	         | j                          d}d}t        |d|       D ]  \  }}	|j                  |      |	j                  |      }	}|j                           | |      }
t	        j
                  |
|	d      }| j                         }|||z  z   }|j                          |j                          |
j                  dd	      }||j                  |	j                  |            j                         j                         z  }||j                         t        |      z  z  } |t        |j                        z  }d
|z  t        |j                        z  }t!        d| d|dd|dd       ||fS )Ng        r   zEpoch descmean)	reductionr>   T)dimkeepdim      Y@zTrain Epoch z: Loss=z.4fz, Accuracy=.2f%)trainr   to	zero_gradr&   cross_entropyr`   backwardstepargmaxeqview_asr"   itemlendatasetprint)modeldevicetrain_loader	optimizerepochbeta
total_losstotal_correctdatatargetoutputnllr%   losspredavg_lossaccs                    r   train_epochr      sY   	KKMJM\&0@A .fwwv		&(9f tooff?]]_TBY}}D}1!56::<AACCdiikCI--
.  C 4 455H

\%9%9!:
:C	LwxnKCy
JKS=r   c           	         | j                          d}t        j                         5  t        |d      D ]  \  }}|j	                  |      |j	                  |      }}t        j
                  |j                  d      d      j	                  |      }t        |      D ]"  }|t        j                   | |      d      z  }$ ||z  }|j                  d      }	||	j                  |      j                         j                         z  } 	 ddd       d|z  t        |j                        z  }
t!        d	| d
|
dd       |
S # 1 sw Y   9xY w)zCPredict using multiple Monte Carlo samples (gas phase aggregation).r   Testingrb   rR   r>   )rf   Nrh   zTest accuracy (MC z samples): ri   rj   )evalr   no_gradr   rl   zerosr]   ranger&   softmaxrq   rr   r"   rt   ru   rv   rw   )rx   ry   test_loadernum_samplescorrectr   r   outputs_r   r   s              r   testr      s*   	JJLG	 	4 9= 	4LD&776?FIIf,=&Dkk$))A,366v>G;' 9199U4[a889{"G>>a>(Dtwwv**,1133G	4	4 .3{223
3C	{m;s3iq
ABJ	4 	4s   C#EEc            	      8   t        j                  t         j                  j                         rdnd      } t	        d|         t        j                  t        j                  dd      t        j                         t        j                         t        j                  dd      g      }t        j                  t        j                         t        j                  dd      g      }t        j                  j                  d	d
d
|      }t        j                  j                  d	dd
|      }t         j                  j                  j!                  |dd
d      }t         j                  j                  j!                  |ddd      }t#        d      j%                  |       }t'        j(                  |j+                         d      }t-        dd      D ]+  }	t/        || |||	d       |	dz  dk(  st1        || |d       - t	        d       t1        || |d       y )NcudacpuzUsing device: rN      )rE   )gHPs?gec]?g~jt?)gۊe?ggDio?g|?5^?z../dataT)rootrk   download	transformF   r   )
batch_sizeshufflenum_workersr	   rQ   gMbP?)lrr>      )r}      r   rR   )r   z.
Final evaluation with 50 Monte Carlo samples:2   )r   ry   r   is_availablerw   
transformsCompose
RandomCropRandomHorizontalFlipToTensor	NormalizetorchvisiondatasetsCIFAR10utilsr   
DataLoaderrK   rl   optimAdam
parametersr   r   r   )
ry   transform_traintransform_test	train_settest_setrz   r   rx   r{   r|   s
             r   mainr      s   \\EJJ$;$;$=&5IF	N6(
#$ ((b!,'')57OP	* O  ''57OP) N
 $$,,)4RVbq,rI##++%RVbp+qH;;##..ySRVde.fL++""--h3PUcd-eKC(++F3E

5++-$7Iq" =E6<EL19><= 

;<4r   __main__r3   )rR   )r   torch.nnr   torch.nn.functional
functionalr&   torch.optimr   r   torchvision.transformsr   r   numpynpModuler   r<   rK   r   r   r   r5    r   r   <module>r      sy          +  
$!RYY $!NKRYY KD$")) $T4&5B zF r   