
    |Si"                     &
   d dl Z d dlmZ d dlmZ d dlmZmZ d dlm	Z	m
Z
 d dlmc mZ 	 e j                  j                  d       dZd Zd Z G d dej(                        Z G d	 d
ej(                        Zd?dZd Zd Zedk(  r e j6                  e j8                  j;                         rdnd      Z ede         ede         e
j>                   e
j@                  d       e
jB                  d       e
jD                          e
jF                  dd      g      Z$ e
j>                   e
j@                  d       e
jB                  d       e
jD                          e
jF                  dd      g      Z% ed        e	jL                  ddde$      Z' e	jL                  dde$      Z( ee'd d!g      \  Z)Z* ed"        e	jV                  ddde%      Z, e	jV                  dde%      Z- ed#        edd$      j]                  e      Z/ ee)d%de&      Z0 ee,d%de&      Z1 ejd                         Z3 ejh                  e/jk                         d'(      Z6ejn                  jq                  e6d)d*+      Z9e/ju                          d,Z; e<e;      D ]"  Z= ed-e=dz    d.e;        d/\  Z>Z?Z@ eAe0e1      ZB eCeB      D ]  \  ZD\  \  ZEZF\  ZGZHeEj]                  e      eFj]                  e      cZEZFeGj]                  e      eHj]                  e      cZGZHe6j                           e/eE      ZJ e3eJeF      ZK e/eG      ZL e3eLeH      ZMd*eKeMz   z  ZNeNj                          e6j                          e>eNj                         z  Z>eJj                  d0      ZSeLj                  d0      ZTe?eSj                  eF      j                         j                         z  Z?e?eTj                  eH      j                         j                         z  Z?e@eFj                  d       eHj                  d       z   z  Z@eDd1z  d k(  sw ed2eD d. eX eYe0       eYe1             d3eNj                         d4        e9j                          d5e?z  e@z  ZZe> eX eYe0       eYe1            z  Z[ ed6e=dz    d.e; d7e[d4d8eZd9d:	       %  ed;        ee(d<de&      Z\ ee-d<de&      Z] ee/e\e      Z^ ee/e]e      Z_ ed=e^d9d:        ed>e_d9d:       yy# e$ r Y w xY w)@    N)
DataLoaderrandom_split)datasets
transformsfile_systemg333333?c                 .    t        | t        z        xs dS N   )intSCALE)xs    ex01.pyscaler      s    q5y>Q    c                   *     e Zd ZdZd fd	Zd Z xZS )
BasicBlockr
   c                    t         |           t        j                  ||d|dd      | _        t        j
                  |      | _        t        j                  ||dddd      | _        t        j
                  |      | _        || _	        y )N   r
   Fstridepaddingbias)
super__init__nnConv2dconv1BatchNorm2dbn1conv2bn2
downsample)self	in_planesplanesr   r"   	__class__s        r   r   zBasicBlock.__init__   sj    YYy&!FATYZ
>>&)YYvvqAER
>>&)$r   c                    |}t        j                  | j                  | j                  |                  }| j	                  | j                  |            }| j                  r| j                  |      }||z  }t        j                  |      S )N)Frelur   r   r!   r    r"   )r#   r   identityouts       r   forwardzBasicBlock.forward!   sh    ffTXXdjjm,-hhtzz#'??q)Hxvvc{r   )r
   N)__name__
__module____qualname__	expansionr   r,   __classcell__r&   s   @r   r   r      s    I%r   r   c                   .     e Zd Zd fd	ZddZd Z xZS )ResNetc                    t         |           t        d      | _        t	        j
                  |t        d      dddd      | _        t	        j                  t        d            | _        | j                  |t        d      |d   d      | _
        | j                  |t        d      |d   d	      | _        | j                  |t        d
      |d	   d	      | _        t	        j                  d      | _        t	        j                  t        d
      |j                   z  |      | _        | j%                         D ]  }t'        |t        j
                        r-t        j(                  j+                  |j,                  dd       Jt'        |t        j                        set        j(                  j/                  |j,                  d       t        j(                  j/                  |j0                  d        y )N@   r   r
   Fr   r   )r            )r
   r
   fan_outr)   )modenonlinearity)r   r   r   r$   r   r   r   r   r   _make_layerlayer1layer2layer3AdaptiveAvgPool2davgpoolLinearr0   fcmodules
isinstanceinitkaiming_normal_weight	constant_r   )r#   blocklayersnum_classesin_channelsmr&   s         r   r   zResNet.__init__,   sl   r YY{E"IqATYZ
>>%), &&ueBi1&M&&ueCj&)A&N&&ueCj&)A&N ++F3))E#J8+F  	-A!RYY'''yv'VAr~~.!!!((A.!!!&&!,	-r   c           	         d }|dk7  s| j                   ||j                  z  k7  ret        j                  t        j                  | j                   ||j                  z  d|d      t        j
                  ||j                  z              } || j                   |||      g}||j                  z  | _         t        d|      D ]$  }|j                   || j                   |             & t        j                  | S )Nr
   F)r   r   )r$   r0   r   
Sequentialr   r   rangeappend)r#   rK   r%   blocksr   r"   rL   _s           r   r=   zResNet._make_layerE   s    
Q;$..FU__,DD		$..&5??*BAf[`av78J
 
CD%//1q&! 	9AMM%78	9 }}f%%r   c                 >   t        j                  | j                  | j                  |                  }| j	                  |      }| j                  |      }| j                  |      }| j                  |      }t        j                  |d      }| j                  |      S r	   )r(   r)   r   r   r>   r?   r@   rB   torchflattenrD   )r#   r   s     r   r,   zResNet.forwardT   ss    FF488DJJqM*+KKNKKNKKNLLOMM!Qwwqzr   )
   r
   )r
   )r-   r.   r/   r   r=   r,   r1   r2   s   @r   r4   r4   +   s    -2&r   r4   r
   rY   c                 *    t        t        g d||       S )N)r8   r8   r8   )r4   r   rN   rM   s     r   resnet18r\   ^   s    *ikBBr   c                    t        j                         }t        j                  | j	                         d      }t        j
                  j                  |dd      }| j                          t        |      D ]c  }d\  }}	}
t        |      D ]
  \  }\  }}|j                  |      |j                  |      }}|j                           | |      } |||      }|j                          |j                          ||j                         z  }|j                  d      }|	|j!                  |      j#                         j                         z  }	|
|j%                  d	      z  }
|d
z  d	k(  st'        d| dt)        |       d|j                         d        |j                          d|	z  |
z  }t'        d|dz    d| d|t)        |      z  dd|dd	       f | S )NMbP?lr         ?	step_sizegammar   r   r   r
   dimr        Batch /	 - Loss: .4f      Y@  Epoch : Loss=, Acc=.2f%)r   CrossEntropyLossoptimAdam
parameterslr_schedulerStepLRtrainrR   	enumerateto	zero_gradbackwardstepitemargmaxeqsumsizeprintlen)modeltrain_loaderepochsdevice	criterion	optimizer	schedulerepoch
total_losscorrecttotal	batch_idxdatatargetoutputlosspredaccs                     r   train_modelr   d   s   ##%I

5++-%8I"")))q)LI	KKMv f%,"
GU)2<)@ 	\%I~f776?FIIf,=&D!4[FVV,DMMONN$))+%J==Q='Dtwwv**,1133GV[[^#E3!#1S->,?yUXHYZ[	\" 	Wnu$q	6('*S=N2Ns1SSYZ]^aYbbcde+f. Lr   c                    | j                          d\  }}t        j                         5  |D ]  \  }}|j                  |      |j                  |      }} | |      }|j	                  d      }||j                  |      j                         j                         z  }||j                  d      z  } 	 d d d        d|z  |z  S # 1 sw Y   xY w)N)r   r   r
   rg   r   rn   )	evalrW   no_gradr|   r   r   r   r   r   )	r   loaderr   r   r   r   r   r   r   s	            r   evaluater      s    	JJLNGU	 $" 	$LD&776?FIIf,=&D4[F==Q='Dtwwv**,1133GV[[^#E	$$ '>E!!$ $s   BCC__main__cudacpuzUsing device: zResNet scale:     )num_output_channels)g_)Ǻ?)gGr?)rb   z
=== Loading MNIST ===z../dataT)rz   download	transformF)rz   r   iP  i'  z=== Loading CIFAR10 ===z@
========== Phase 1: Joint Training (MNIST + CIFAR10) ==========r[   r7   )
batch_sizeshufflenum_workersr^   r_   r8   rb   rc   r   z
Epoch rk   rf   rg   d   rj   rl   rm   rn   ro   rp   rq   rr   rs   z*
========== Phase 2: Evaluation ==========r9   zMNIST test accuracy: zCIFAR10 test accuracy: )r
   rY   )`rW   torch.nnr   torch.optimru   torch.utils.datar   r   torchvisionr   r   torch.nn.functional
functionalr(   multiprocessingset_sharing_strategyRuntimeErrorr   NUM_WORKERSr   Moduler   r4   r\   r   r   r-   r   r   is_availabler   ComposeResize	GrayscaleToTensor	Normalizemnist_transformcifar_transformMNIST
mnist_full
mnist_testmnist_train	mnist_valCIFAR10
cifar_full
cifar_testr|   r   mnist_loadercifar_loaderrt   r   rv   rw   r   rx   ry   r   rz   epochs_jointrR   r   r   r   r   zippaired_batchesr{   r   data_mtarget_mdata_ctarget_cr}   out_mloss_mout_closs_cr   r~   r   r   r   pred_mpred_cr   r   r   minr   r   avg_lossmnist_test_loadercifar_test_loader	mnist_acc	cifar_acc r   r   <module>r      sk      5 ,  		..}=
 	
 *0RYY 0fC@
" zU\\EJJ$;$;$=&5IF	N6(
#$	N5'
"# )j((
"
3

Y	2	* O )j((
"
3

VV,	* O 

#$	P_`J	/RJ)*uenEK	
#$!!!)4$RabJ!!!)5OTJ 

MN366v>Ekc4U`aLjS$T_`L###%I

5++-%8I"")))q)LI	KKML|$ 'Xq	<.12%,"
GU\<8CL^C\ 	?I?*,>VX%yy0(++f2EFH%yy0(++f2EFH!&MEuh/F&MEuh/F&6/*DMMONN$))+%J\\a\(F\\a\(Fvyy*..05577Gvyy*..05577GX]]1%a(888E3!#yk3s</@#lBS+T*U V#yy{3/13	< 	go%C$5s<7H IIq	<.~VCPS9TUVWO'XR 

78":#uZef":#uZef 16:I 16:I	!)C
23	#Ic?!
45E S  		s   T TT