
    iY5                        d Z ddlZddlmZ ddlmc mZ ddlmZ ddl	m
Z
 ddlZddlmZ ddlZ ej                  ej                   j#                         rdnd      ZdZdZdZg d	Zd
Z G d dej0                        Z G d dej0                        Z G d d      Z G d dej0                        Z G d dej0                        Zd Zd ZddZ ddZ!d Z"e#dk(  r e"       Z$yy)u   
Leaps of Intelligence: ResNet18 → MLP via Lottery Ticket Hypothesis
===================================================================
    N)
DataLoadercudacpu         )g      ?gffffff?g?g?z./datac                   2     e Zd Zd fd	Zd Zd Zd Z xZS )ModifiedResNet18c                    t         |           t        j                  dddddd      | _        t        j
                  d      | _        | j                  dddd      | _        | j                  dddd      | _	        | j                  dd	dd      | _
        | j                  d	d
dd      | _        t        j                  d      | _        t        j                  d
|      | _        y )Nr   @      F)kernel_sizestridepaddingbiasr   )r   r         )r   r   )super__init__nnConv2dconv1BatchNorm2dbn1_make_layerlayer1layer2layer3layer4AdaptiveAvgPool2davgpoolLinearfc)selfnum_classes	__class__s     c08.pyr   zModifiedResNet18.__init__   s    YYq"!AquU
>>"%&&r2q&;&&r3!&<&&sC1&=&&sC1&=++F3))C-    c                     g }|j                  t        |||             t        d|      D ]  }|j                  t        ||d               t        j                  | S Nr   )append
BasicBlockranger   
Sequential)r$   in_chout_chblocksr   layers_s          r'   r   zModifiedResNet18._make_layer)   sV    j78q&! 	9AMM*VVQ78	9}}f%%r(   c                 `   t        j                  | j                  | j                  |                  }| j	                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }t        j                  |d      }| j                  |      S r*   )Frelur   r   r   r   r   r   r!   torchflattenr#   r$   xs     r'   forwardzModifiedResNet18.forward0   s    FF488DJJqM*+KKNKKNKKNKKNLLOMM!Qwwqzr(   c                 B    t        d | j                         D              S )Nc              3   <   K   | ]  }|j                           y wNnumel).0ps     r'   	<genexpr>z3ModifiedResNet18.get_param_count.<locals>.<genexpr>;   s     817798   )sum
parameters)r$   s    r'   get_param_countz ModifiedResNet18.get_param_count:   s    8doo&7888r(   )
   )__name__
__module____qualname__r   r   r;   rG   __classcell__r&   s   @r'   r
   r
      s    
.&9r(   r
   c                   (     e Zd ZdZ fdZd Z xZS )r,   r   c           	         t         |           t        j                  ||d|dd      | _        t        j
                  |      | _        t        j                  ||dddd      | _        t        j
                  |      | _        t        j                         | _
        |dk7  s||k7  rGt        j                  t        j                  ||d|d      t        j
                  |            | _
        y y )Nr   r   Fr   )r   r   r   r   r   r   r   conv2bn2r.   shortcut)r$   r/   r0   r   r&   s       r'   r   zBasicBlock.__init__A   s    YYufaG
>>&)YYvvq!QUC
>>&)Q;%6/MM		%F?v&DM *r(   c                     t        j                  | j                  | j                  |                  }| j	                  | j                  |            }|| j                  |      z  }t        j                  |      S r>   )r5   r6   r   r   rR   rQ   rS   )r$   r:   outs      r'   r;   zBasicBlock.forwardO   sX    ffTXXdjjm,-hhtzz#'t}}Qvvc{r(   )rI   rJ   rK   	expansionr   r;   rL   rM   s   @r'   r,   r,   >   s    Ir(   r,   c                   2    e Zd ZdZed        Zedd       Zy)LeapsExtractorz
    Extract the 'leaps of intelligence' from a trained/pruned ResNet.
    The key insight: each layer-to-layer connection can be expressed as 
    a linear projection, even if originally a convolution.
    c                 ^    | j                   j                  j                  j                         S )zExtract the FC layer weights)r#   weightdataclone)resnets    r'   extract_fc_layersz LeapsExtractor.extract_fc_layers`   s#     yy$$**,,r(   c                 l   |dk(  r4| j                   j                  j                  }|j                  d      }|S |j	                  d      rjt        | |      }d}|j                         D ]  }t        |t        j                        s|}! |r(|j                  j                  }|j                  d      S y)z
        Convert conv weights to FC by unfolding spatial dimensions.
        This preserves the learned 'leaps' as direct connections.
        r   )r   r   dimlayerN)
r   rZ   r[   mean
startswithgetattrmodules
isinstancer   r   )r]   
layer_name
in_spatialww_pooledrb   	last_convms           r'   extract_conv_as_fcz!LeapsExtractor.extract_conv_as_fce   s      ##((Avv&v)HO""7+FJ/EI]]_ "a+ !I" $$))vv&v))r(   N)   )rI   rJ   rK   __doc__staticmethodr^   rn    r(   r'   rX   rX   Y   s/     - -  r(   rX   c                   6     e Zd ZdZd fd	Zd Zd Zd Z xZS )LeapsMLPu   
    MLP that uses the leap patterns learned by ResNet.
    Architecture matches the layer-to-layer flow: 
    Input → [Leap1] → [Leap2] → [Leap3] → Output
    c                 
   t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y )NFrP   )r   r   r   r"   leap1leap2leap3leap1_br$   
input_sizehidden1hidden2outputr&   s        r'   r   zLeapsMLP.__init__   s`    YYz7?
YYwe<
YYwU;
 yyW5Ar(   c                 N   |j                  |j                  d      d      }t        j                  | j	                  |            }t        j                  | j                  |            }t        j                  | j                  |            }| j                  ||dz  z         }|S )Nr   g333333?)viewsizer5   r6   rv   ry   rw   rx   )r$   r:   h1skiph2rU   s         r'   r;   zLeapsMLP.forward   s}    FF166!9b! VVDJJqM"vvdll1o& VVDJJrN# jjdSj)
r(   c                    t        j                         5  |j                  j                  j                  }| j                  || j                  j                        }|| j                  j                  j                  d|j                  d      d|j                  d      f<   |j                  j                  j                  }t        | j                  j                  j                  d      |j                  d            }t        | j                  j                  j                  d      |j                  d            }|d|d|f   | j                  j                  j                  d|d|f<   ddd       y# 1 sw Y   yxY w)z/Inject learned leap patterns from pruned ResNetNr   r   )r7   no_gradr   rZ   r[   _extract_top_leapsrv   out_featuresr   r#   minrx   )r$   r]   prune_ratioconv_wleap_wfc_wr   in_featuress           r'   inject_leapszLeapsMLP.inject_leaps   s3   ]]_ 	d \\((--F,,VTZZ5L5LMFGMDJJ""?FKKN?OV[[^O#CD 99##((Dtzz0055a8$))A,GLdjj//44Q71FKBF}}VbWbVbGbBcDJJ""=L=,;,#>?	d 	d 	ds   EE::Fc                    |j                         j                         }t        |dz  |j                               }|dk  r$|j	                  d|d   j                         f      S ||j                         k\  r*|j                  |j                  d      d      d|ddf   S |j                  |      \  }}t        j                  |      }d||<   |j                  |      }||z  j                  |j                  d      d      d|ddf   S )z2Extract the highest-magnitude connections as leapsd   r   r   Nr   )absr8   r   r@   	new_zerosr   r   topkr7   
zeros_likeview_as)r$   conv_weights
target_outflatkr3   indicesmasks           r'   r   zLeapsMLP._extract_top_leaps   s     !))+
S $**,/6))1l1o.C.C.E*FGG

$$\%6%6q%92>{
{A~NNYYq\
7%W||L)t#)),*;*;A*>CKZKQRNSSr(     r   r   rH   )	rI   rJ   rK   rp   r   r;   r   r   rL   rM   s   @r'   rt   rt   ~   s    
Bd Tr(   rt   c                   &     e Zd Zd fd	Zd Z xZS )	RandomMLPc                     t         |           t        j                  ||      | _        t        j                  ||      | _        t        j                  ||      | _        y r>   )r   r   r   r"   fc1fc2fc3rz   s        r'   r   zRandomMLP.__init__   sD    99Z199Wg.99Wf-r(   c                     |j                  |j                  d      d      }t        j                  | j	                  |            }t        j                  | j                  |            }| j                  |      S )Nr   r   )r   r   r5   r6   r   r   r   r9   s     r'   r;   zRandomMLP.forward   sT    FF166!9b!FF488A;FF488A;xx{r(   r   )rI   rJ   rK   r   r;   rL   rM   s   @r'   r   r      s    .r(   r   c                    i }| j                         D ]  \  }}d|v s|j                         dk\  s|j                  j                         j	                         }t        |j                         d|z
  z        }|dkD  r|j                  |      d   nd}|j                  j                         |kD  j                         }|||<   |xj                  |z  c_         | |fS )zPrune lowest magnitude weightsrZ   r   r   r   )	named_parametersra   r[   r   r8   intr@   kthvaluefloat)	modelr   masksnameparamr   r   	thresholdr   s	            r'   magnitude_pruner      s    E--/ et		q 0::>>#++-DDJJLAO45A/01ua(+!IJJNN$y0779DE$KJJ$J %<r(   c                 J   | j                         D ]  }t        |t        j                  t        j                  f      s.t        j
                  j                  |j                  dd       |j                  gt        j
                  j                  |j                  d        y)z,Reset to small random (lottery ticket style)fan_outr6   )modenonlinearityNr   )
rf   rg   r   r   r"   initkaiming_normal_rZ   r   	constant_)r   rm   s     r'   reset_weightsr      sm    ]]_ -a"))RYY/0GG##AHH96#Rvv!!!!&&!,	-r(   c                    t        j                  | j                         |      }t        j                         }| j                          t        |      D ]  }d\  }	}
}|D ]  \  }}|j                  |      |j                  |      }}|j                           | |      } |||      }|j                          |j                          ||j                         z  }|j                  d      }|	|j                  |      j                         j                         z  }	|
|j                  d      z  }
 d|	z  |
z  }t!        d| d|dz    d	| d
|t#        |      z  dd|dd        | S )N)lr)r   r   r   r   r`   r   r     z Ep /z	 - Loss: z.4fz, Acc: .2f%)optimAdamrF   r   CrossEntropyLosstrainr-   to	zero_gradbackwardstepitemargmaxeqrE   r   printlen)r   loaderepochsdevicer   r   	optimizer	criterionepochcorrecttotalloss_sumr[   targetrU   losspredaccs                     r'   r   r      sa   

5++-"5I##%I	KKMv e#* " 	$LD&776?FIIf,=&D!+CS&)DMMONN		#H::!:$Dtwwv**,1133GV[[^#E	$ Gme#4&U1WIQvhiV8LS7QQXY\]`Xaabcd!e$ Lr(   c                    | j                          d\  }}t        j                         5  |D ]  \  }}|j                  |      |j                  |      }} | |      }|j	                  d      }	||	j                  |      j                         j                         z  }||j                  d      z  } 	 d d d        d|z  |z  }
t        d| d|
dd	       |
S # 1 sw Y   &xY w)
N)r   r   r   r`   r   r   r   z Test: r   r   )
evalr7   r   r   r   r   rE   r   r   r   )r   r   r   r   r   r   r[   r   rU   r   r   s              r'   testr     s    	JJLNGU	 $" 	$LD&776?FIIf,=&D+C::!:$Dtwwv**,1133GV[[^#E	$$ -%
C	BtfGC9A
&'J$ $s   BCC$c                     t        d       t        d       t        d       t        dt                t        j                  t        j                         t        j
                  dd      g      } t        j                  j                  t        dd|       }t        j                  j                  t        dd|       }t        |t        d	      }t        |t        
      }t        dt        |       dt        |       d       t        d       t               j                  t              }t        d|j                         d       t!        j                          }t#        ||t$        t        d      }t!        j                          |z
  }t'        ||t        d      }t        d|dd       t        d       g }	t(        D ]  }
t        d|
dz  dd       t               j                  t              }t#        ||dt        d|
dz  dd       t+        ||
      \  }}t-        d |j/                         D              }t-        d |j/                         D              }t        d|d d!||z  dz  dd"       t1        d#d$d%d&'      j                  t              }|j3                  ||
       t        d(       t#        ||t4        t        d)      }t'        ||t        d)      }t7        d#d$d%d&'      j                  t              }t#        ||t4        t        d*      }t'        ||t        d*      }|	j9                  |
||||d+       t                 t        d       t        d,       t        d       t        d-d.d/dd0d/d)d0d/d*d0d/d1d0	       t        d2       |	D ]2  }t        |d3   dz  d4d5|d6   d7d5|d8   d7d5|d9   d7d5|d:   d;	       4 t        d2       t        d<       |	S )=Nz<============================================================u:   LEAPS OF INTELLIGENCE: ResNet18 → MLP via Lottery TicketzDevice: )g_)Ǻ?)gGr?T)r   download	transformF)
batch_sizeshuffle)r   zTrain: z, Test: 
z"1. Training ResNet18 (scaffold)...z   Parameters: ,ResNet18z	   Time: z.1fzs
z&2. Finding 'Leaps of Intelligence'...
z  === Prune ratio: r   z.0fz% ===   zPruned-r   c              3   X   K   | ]"  }|j                         j                          $ y wr>   )rE   r   rA   rm   s     r'   rC   zmain.<locals>.<genexpr>C  s     >1>s   (*c              3   <   K   | ]  }|j                           y wr>   r?   r   s     r'   rC   zmain.<locals>.<genexpr>D  s     7Aaggi7rD   z  Remaining params: z,.0fz (z%)r   r   r   rH   )r{   r|   r}   r~   z  Training LeapsMLP...rt   r   )pruner]   leapsrandom	remainingzRESULTS SUMMARYzPrune%z>8z | z>10Paramsz<------------------------------------------------------------r   z>7.0fz% | r]   z>10.2fr   r   r   z>10,.0fzA
If LeapsMLP > RandomMLP: 'leaps of intelligence' were extracted!)r   DEVICE
transformsComposeToTensor	NormalizetorchvisiondatasetsMNIST
MNIST_PATHr   
BATCH_SIZEr   r
   r   rG   timer   EPOCHS_CONVr   PRUNE_RATIOSr   rE   valuesrt   r   
EPOCHS_MLPr   r+   )r   	train_settest_settrain_loadertest_loaderr]   startresnet_time
resnet_accresultsr   resnet_pr3   r   r   total_p	leaps_mlp	leaps_acc
random_mlp
random_accrs                        r'   mainr    s   	&M	
FG	&M	HVH
 ""Y	2$ I $$**:TD\e*fI##))*ED\e)fHiJMLX*=K	GC	N#8CM?"
=> 

./""6*F	OF224Q7
89IIKE6<fjIF))+%Kfk6:>J	Ik#&c
*+ 

34G# "#KOC#8>? $%((0ha7;s?3:Oq1QR "(K84>>>	777$Yt$4By7H7LS6QQSTU S#bQTTU[\	x5 	&()\:vzR	KD	 #sCPRSVVW]^
:|ZU
*k6;G
   "
 	 	E"J 
&M	
	&M	XbMZ,C
3/?s;sBSSVW_`cVd
ef	&M L7C&d1X;v*>d1W:fBUUYZ[\dZeflYmmqrst  sA  BI  rJ  K  	LL	&M	
NONr(   __main__)ModelgMbP?)r  )%rp   r7   torch.nnr   torch.nn.functional
functionalr5   torch.optimr   torch.utils.datar   r   torchvision.transformsr   r   r   r   is_availabler   r   r   r   r   r   Moduler
   r,   rX   rt   r   r   r   r   r   r  rI   r  rr   r(   r'   <module>r     s   
      '  + 
 


 7 7 9fu	E

#

9ryy 9D 6" "J<Tryy <TD		 "-4&Qh zfG r(   