
    Εi<                     (   d Z ddlZddlmZ ddlmc mZ ddlmZ ddl	Z	ddl
mZ ddlmZ ddlZddlZ G d dej                         Z G d dej                         Z G d d	ej                         Zdd
ZddZddZddZd Zedk(  r e        yy)u  
Hyper‑Gas CIFAR‑10 Classifier with Gas Relaxation and Compression

This script trains a Bayesian (Hyper‑Gas) CNN where each weight is a Gaussian distribution.
After training, we apply a gas‑relaxation step (diffusion + condensation) to rearrange the
weight distributions into a more compressible state. Then we quantize the weight means to 8‑bit
integers (int8) and optionally discard the variance parameters for high‑confidence weights.
The result is a compressed model that retains predictive accuracy while drastically reducing
memory footprint – exactly as a gas naturally arranges itself before being compressed.

Usage:
    python hypergas_cifar10.py
    N)tqdmc                   &     e Zd Zd fd	Zd Z xZS )HyperGasLinearc                 4   t         |           || _        || _        t	        j
                  t        j                  ||      j                  dd            | _	        t	        j
                  t        j                  ||      j                  dd            | _
        t	        j
                  t        j                  |      j                               | _        t	        j
                  t        j                  |      j                  dd            | _        || _        y )Nr   {Gz?皙?)super__init__in_featuresout_featuresnn	ParametertorchTensornormal_w_muw_log_sigmazero_b_mub_log_sigmaprior_sigma)selfr   r   r   	__class__s       cifar10_hypergas2.pyr   zHyperGasLinear.__init__   s    &(LLlK!H!P!PQRTX!YZ	<<\;(O(W(WXZ\_(`aLLl!;!A!A!CD	<<\(B(J(J2s(ST&    c                 |   t        j                  | j                        }t        j                  | j                        }t        j                  | j
                        }t        j                  | j                        }| j                  ||z  z   }| j                  ||z  z   }t        j                  | j
                  t        j                  t        j                  | j                              z
  |dz  | j                  dz  z   d| j                  dz  z  z  z   dz
        }t        j                  | j                  t        j                  t        j                  | j                              z
  |dz  | j                  dz  z   d| j                  dz  z  z  z   dz
        }	||	z   | _        t        j                  |||      S )N         ?)r   
randn_liker   r   expr   r   sumlogtensorr   klFlinear
r   xw_epsb_epsw_sigmab_sigmawbkl_wkl_bs
             r   forwardzHyperGasLinear.forward)   sl     +  +))D,,-))D,,-II%'II%'yy))EIIell4CSCS6T,UU!1*tyy!|3D<L<La<O8OPQSVW Xyy))EIIell4CSCS6T,UU!1*tyy!|3D<L<La<O8OPQSVW X+xx1a  r   r	   __name__
__module____qualname__r   r2   __classcell__r   s   @r   r   r      s    	'!r   r   c                   &     e Zd Zd fd	Zd Z xZS )HyperGasConv2dc                    t         |           || _        || _        t	        |t
              r||fn|| _        || _        || _        || j                  d   z  | j                  d   z  }t        j                  t        j                  ||g| j                   j                  dd            | _        t        j                  t        j                  ||g| j                   j                  dd            | _        t        j                  t        j                  |      j!                               | _        t        j                  t        j                  |      j                  dd            | _        || _        y )Nr      r   r   r	   )r
   r   in_channelsout_channels
isinstanceintkernel_sizestridepaddingr   r   r   r   r   r   r   r   r   r   r   )	r   r>   r?   rB   rC   rD   r   fan_inr   s	           r   r   zHyperGasConv2d.__init__:   s(   &(9CKQT9UK5[ft//22T5E5Ea5HHLLlK![$JZJZ![!c!cdegk!lm	<<\;(bQUQaQa(b(j(jkmor(stLLl!;!A!A!CD	<<\(B(J(J2s(ST&r   c                    t        j                  | j                        }t        j                  | j                        }t        j                  | j
                        }t        j                  | j                        }| j                  ||z  z   }| j                  ||z  z   }t        j                  | j
                  t        j                  t        j                  | j                              z
  |dz  | j                  dz  z   d| j                  dz  z  z  z   dz
        }t        j                  | j                  t        j                  t        j                  | j                              z
  |dz  | j                  dz  z   d| j                  dz  z  z  z   dz
        }	||	z   | _        t        j                  |||| j                  | j                        S )Nr   r   rC   rD   )r   r    r   r   r!   r   r   r"   r#   r$   r   r%   r&   conv2drC   rD   r(   s
             r   r2   zHyperGasConv2d.forwardI   sx     +  +))D,,-))D,,-II%'II%'yy))EIIell4CSCS6T,UU!1*tyy!|3D<L<La<O8OPQSVW Xyy))EIIell4CSCS6T,UU!1*tyy!|3D<L<La<O8OPQSVW X+xx1aT\\JJr   )r=   r   r	   r4   r9   s   @r   r;   r;   9   s    'Kr   r;   c                   .     e Zd Zd fd	ZddZd Z xZS )HyperGasNetc                     t         |           t        dddd|      | _        t        dddd|      | _        t        j                  dd      | _        t        dd|	      | _	        t        dd
|	      | _
        y )N       r=   )rD   r   @   r   i      r   
   )r
   r   r;   conv1conv2r   	MaxPool2dpoolr   fc1fc2)r   r   r   s     r   r   zHyperGasNet.__init__Z   sh    #Ar1a[Q
#BAqkR
LLA&	!*c{K!#r{Cr   c           	         |st        j                  t        j                  || j                  j                  | j                  j
                  dd            }| j                  |      }t        j                  t        j                  || j                  j                  | j                  j
                  dd            }| j                  |      }|j                  |j                  d      d      }t        j                  t        j                  || j                  j                  | j                  j
                              }t        j                  || j                  j                  | j                  j
                        }|S t        j                  | j                  |            }| j                  |      }t        j                  | j                  |            }| j                  |      }|j                  |j                  d      d      }t        j                  | j                  |            }| j                  |      }|S )Nr=   rG   r   )r&   relurH   rR   r   r   rU   rS   viewsizer'   rV   rW   )r   r)   samples      r   r2   zHyperGasNet.forwardb   so   qxx4::??DJJOOAWXYZA		!Aqxx4::??DJJOOAWXYZA		!Aqvvay"%Aqxx488==$((--@AADHHMM488==9AHtzz!}%A		!Atzz!}%A		!Aqvvay"%Atxx{#AAHr   c                     | j                   j                  | j                  j                  z   | j                  j                  z   | j                  j                  z   dz  S )Ng     L@)rR   r%   rS   rV   rW   )r   s    r   kl_losszHyperGasNet.kl_lossw   s<    



-;dhhkkIWTTr   r3   )T)r5   r6   r7   r   r2   r_   r8   r9   s   @r   rJ   rJ   Y   s    D*Ur   rJ   c           	      |   t        j                         5  t        |      D ]  }| j                         D ]  }t	        |d      st	        |d      s|j
                  }|j                  }t        j                  |      }|j                         }	t        j                  |      |z  }
|||	|z
  z  z   |
z   }||j
                  _
        t        j                  ||	z
        }|ddt        j                  |      z  z   z  }t        j                  |dz         |j                  _
          	 ddd       y# 1 sw Y   yxY w)z
    Rearrange the weight distributions like a gas:
    - Means diffuse toward local averages (pressure equalisation).
    - Variances shrink near clusters (condensation) and expand elsewhere.
    This prepares the numbers for compression.
    r   r   g?r	   g:0yE>N)r   no_gradrangemoduleshasattrr   r   r!   meanr    dataabstanhr#   )modelstepstemperaturelrstepmodulemu	log_sigmasigma	local_avgnoisemu_newdistance	sigma_news                 r   	relax_gasrw   ~   s    
 J%L 	JD--/ J66*wv}/MB & 2 2I!IIi0E !#	I ",,R0;>E"	B"77%?F'-FKK$  %yy));<H %sUZZ5I/I)I JI.3ii	D8H.IF&&+#J	JJ J Js   /D2D2CD22D;c           	      x	   i }t        | d      r| j                  j                  nd}t        |      j	                  t        | j                               j                        }|j                  | j                                d}d}|j                         D ]c  \  }}t        |d      s|j                  j                         t        |d      r|j                  j                         ndz   }	||	z  }|j                  j                         j                         j!                         }
|
j#                         |
j%                         }}||z
  dk  r
|dz
  |dz   }}t'        j(                  |
|z
  ||z
  z  d	z        j+                  t&        j,                        }||||
j.                  d
d||dz   <   ||j0                  dz  z  }t        |d      rNt3        j4                  |j6                        j                         j                         j!                         }||k  }t        |d      s|j                  j                         j                         j!                         }|j#                         |j%                         }}||z
  dk  r
|dz
  |dz   }}t'        j(                  ||z
  ||z
  z  d	z        j+                  t&        j,                        }||||j.                  d
d||dz   <   ||j0                  dz  z  }f t3        j8                         5  |j                         D ]U  \  }}|dz   |v r||dz      }|d   j+                  t&        j:                        dz  |d   |d   z
  z  |d   z   }t3        j<                  |j?                  |d         |j                  j                  |j                  j@                        |j                  _!        |dz   |v s||dz      }|d   j+                  t&        j:                        dz  |d   |d   z
  z  |d   z   }t3        j<                  |j?                  |d         |j                  j                  |j                  j@                        |j                  _!        X 	 ddd       |dkD  r|dz  |z  nd}tE        d|dz   d| d|dd       ||fS # 1 sw Y   5xY w)u  
    Compress the Hyper‑Gas model by quantizing weight means to int8.
    Optionally prune weights with very low sigma (deterministic).
    Returns a dictionary of quantized parameters and metadata.
    Also returns a "decompressed" model (using quantized means) for evaluation.
    rR   r	   rP   r   r   r   gư>g      ?   uint8)rf   minmaxshapedtypez.w_mu   r   z.b_murf   g     o@r|   r{   r}   )devicer~   NrM   zCompression: original u    bits → compressed z bits (ratio .2fzx))#rd   rR   r   rJ   tonext
parametersr   load_state_dict
state_dictnamed_modulesr   numelr   detachcpunumpyr{   r|   nproundastyperz   r}   r\   r   r!   r   ra   float32r$   reshaper~   rf   print)ri   prune_sigma_threshold
compressedr   decompressed_modeltotal_original_paramstotal_compressed_bitsnamern   original_paramsro   mu_minmu_maxmu_qrq   
prune_maskr/   b_minb_maxb_qq_infomu_deqb_deqcompression_ratios                           r   compress_modelr      s    J-4UG-D%++))#K$=@@eFVFVFXAYA`A`a&&u'7'7'9:*88: ,6f66"$kk//1GTZ\bLcV[[5F5F5HijkO!_4! ##%))+113BVVXrvvxFF%!'#v|88R&[Vf_=CDKKBHHUD  *Jtg~& "TYY]2! v}-		&"4"45<<>BBDJJL"%::

 vv&KK&&(,,.446 uuwu5=4'#(3;5EhhE	eem<sBCJJ288T  WW$.
4'>* &A5%Y,6^ 
 	D.<<> 	DLD&g~+#D7N3 .//

;eCuX^_dXeHefiopuivv#(<<vg0OX^XcXcXjXjrxr}r}  sD  sD  $E g~+#D7N3..rzz:UBve}W]^cWdGdehnothuu#(<<fWo0NW]WbWbWiWiqwq|q|  rC  rC  $D 	D	D OdfgNg-25JJmn	"#82#=">>STiSjjw  yJ  KN  xO  OQ  R  S)))	D 	Ds   CR0B#R00R9c           	         | j                          d}d}t        |d|       D ]  \  }}	|j                  |      |	j                  |      }	}|j                           | |      }
t	        j
                  |
|	d      }| j                         }|||z  z   }|j                          |j                          |
j                  d      }||j                  |	      j                         j                         z  }||j                         t        |      z  z  } |t        |j                        z  }d	|z  t        |j                        z  }t        d
| d|dd|dd       ||fS )Ng        r   zEpoch descre   )	reductionr=   dim      Y@zTrain Epoch z: Loss=z.4fz, Accuracy=r   %)trainr   r   	zero_gradr&   cross_entropyr_   backwardrm   argmaxeqr"   itemlendatasetr   )ri   r   train_loader	optimizerepochbeta
total_losstotal_correctrf   targetoutputnllr%   losspredavg_lossaccs                    r   train_epochr      sK   	KKMJM\&0@A .fwwv		&(9ftooff?]]_TBY}}}#,,.3355diikCI--
. C 4 455H

\%9%9!:
:C	LwxnKCy
JKS=r   c                 |   | j                          d}t        j                         5  t        |d      D ]  \  }}|j	                  |      |j	                  |      }}t        j
                  |j                  d      d      j	                  |      }t        |      D ]$  } | |      }	|t        j                  |	d      z  }& ||z  }|j                  d      }
||
j                  |      j                         j                         z  } 	 ddd       d|z  t        |j                        z  }|S # 1 sw Y   &xY w)	z9Evaluate accuracy using Monte Carlo sampling (gas phase).r   
Evaluatingr   rQ   r=   r   Nr   )evalr   ra   r   r   zerosr\   rb   r&   softmaxr   r   r"   r   r   r   )ri   r   test_loader
mc_samplescorrectrf   r   outputs_logitsr   r   s               r   evaluater     s   	JJLG	 
4 <@ 		4LD&776?FIIf,=&Dkk$))A,366v>G:& 4t199V334 z!G>>a>(Dtwwv**,1133G		4
4 .3{223
3CJ
4 
4s   C%D22D;c            	      2   t        j                  t         j                  j                         rdnd      } t	        d|         t        j                  t        j                  dd      t        j                         t        j                         t        j                  dd      g      }t        j                  t        j                         t        j                  dd      g      }t        j                  j                  d	d
d
|      }t        j                  j                  d	dd
|      }t         j                  j                  j!                  |dd
d      }t         j                  j                  j!                  |ddd      }t#        d      j%                  |       }t'        j(                  |j+                         d      }t	        d       t-        dd      D ]>  }	t/        || |||	d       |	dz  dk(  st1        || |d      }
t	        d|	 d|
dd       @ t	        d        t1        || |d!      }t	        d"|dd       t	        d#       t3        |dd$d%&       t	        d'       t1        || |d!      }t	        d(|dd       t	        d)       t5        |d%*      \  }}t	        d+       t1        || |d      }t	        d,|dd       t	        d-       t	        d.|dd       t	        d/|dd       t	        d0|dd       t	        d1       y )2Ncudar   zUsing device: rM      )rD   )gHPs?gec]?g~jt?)gۊe?ggDio?g|?5^?z../dataT)rootr   download	transformF   r   )
batch_sizeshufflenum_workersr	   rP   gMbP?)rl   z'
===== TRAINING PHASE (20 epochs) =====r=      )r   rQ   r      )r   u      → Test accuracy after epoch z: r   r   z/
===== FINAL EVALUATION BEFORE RELAXATION =====   z#Test accuracy (before relaxation): z&
===== GAS RELAXATION (10 steps) =====皙?{Gz?)rj   rk   rl   z,
===== EVALUATION AFTER GAS RELAXATION =====z&Test accuracy (after gas relaxation): z
===== COMPRESSION =====)r   z+
===== EVALUATION OF COMPRESSED MODEL =====z,Test accuracy (compressed model, 1 sample): z
===== SUMMARY =====zOriginal model test acc: zAfter gas relaxation: z!After quantization (compressed): u:   Compression done. Model weights quantized to 8‑bit ints.)r   r   r   is_availabler   
transformsCompose
RandomCropRandomHorizontalFlipToTensor	NormalizetorchvisiondatasetsCIFAR10utilsrf   
DataLoaderrJ   r   optimAdamr   rb   r   r   rw   r   )r   transform_traintransform_test	train_settest_setr   r   ri   r   r   r   
acc_beforeacc_after_relaxcompressed_statecompressed_modelacc_compresseds                   r   mainr     s   \\EJJ$;$;$=&5IF	N6(
#$ !((b!,'')57OP	* O  ''57OP) N $$,,)4RVbq,rI##++%RVbp+qH;;##..ySRVde.fL++""--h3PUcd-eK C(++F3E

5++-$7I 

45q" JE6<EL2:?5&+!DC4UG2c#YaHI	J 

<=%DJ	/
3/?q
AB 

34e24D9	
9:ufkbIO	2?32Gq
IJ 

%&)7UY)Z&&	
89.PQRN	88LA
NO	
!"	%j%5Q
78	"?3"7q
9:	-nS-A
CD	
FGr   __main__)rQ   r   r   )r   r3   )rQ   )__doc__r   torch.nnr   torch.nn.functional
functionalr&   torch.optimr   r   torchvision.transformsr   r   r   r   mathModuler   r;   rJ   rw   r   r   r   r   r5    r   r   <module>r     s          +   
!RYY !8KRYY K@U")) UJJ@J*`,,8Hv zF r   