
    q$ij                     T   d Z ddlZddlZddlZddlZddlZddlmZ ddl	mc m
Z ddlmZmZ ddlmZ ddlmZmZmZ ddlmZ ddlmZ  G d d	      Z e       Zd
 Zd ZddZddZ G d de      Zd Z  G d dejB                        Z"d Z#d Z$ddZ%d Z&d Z'e(dk(  r e'        yy)zp
Word-MLP: Conv2D + Dense for Visual Word Embeddings
Skip-gram context head for stronger semantic relationships
    N)Dataset
DataLoader)pad_sequence)Image	ImageDraw	ImageFont)defaultdict)Pathc                       e Zd ZdZdZdZdZdZdZdZ	dZ
dZd	Zd	Z ej                  ej                   j#                         rd
      Zyd      Zy)Configz./books         N   @   MbP?      ?cudacpu)__name__
__module____qualname__book_folderimg_size	font_size	embed_dimnum_classesnum_negatives
batch_sizeepochslr
cls_weightskipgram_weighttorchdevicer   is_available     ex02.pyr   r      si    K HI IKM JF	B JO U\\EJJ$;$;$=&IF5IFr(   r   c                 D   g }t        |       }|j                         st        d|  d       t               S |j	                  d      D ]  }t        d|j
                          t        |ddd      5 }|j                         }d	d	d	       t        j                  d
      }|j                  |D cg c]  }|j                          c}        t        |      dk(  rt        d|  d       t               S t        dt        |       d       |S # 1 sw Y   xY wc c}w )z1Load all .md files from folder and extract words.u   ⚠️ Folder 'z' not found. Using sample text.z*.mdu   📖 Loading: rzutf-8ignore)encodingerrorsN[a-zA-Z]{2,}r   u   ⚠️ No words found in z. Using sample text.u   ✅ Extracted z words from markdown files.)r
   existsprintsample_wordsglobnameopenreadrefindallextendlowerlen)folder_pathwordsfoldermd_filefcontent	extractedws           r)   load_markdown_filesrD   6   s   E+F==?},KLM~;;v& 5w||n-.'3B 	affhG	 JJ8	3Aaggi345 5zQ)+6JKL~	N3u:,&A
BCL	 	 4s   2D0D
D	c                  N    d} t        j                  d| j                               S )z1Fallback sample words if no markdown files found.a  
    intelligence artificial machine learning neural network deep 
    consciousness awareness thinking reasoning planning algorithm 
    data information knowledge wisdom understanding insight logic 
    mathematics physics chemistry biology science research study
    r/   )r7   r8   r:   )sample_texts    r)   r2   r2   N   s%    K ::o{'8'8':;;r(   c                    t        t              }| D ]  }||xx   dz  cc<    t        |j                         d       }|d| D cg c]  \  }}|	 }}}t	        |      D ci c]  \  }}||
 }	}}|	j                         D ci c]  \  }}||
 }
}}t        dt        |              |	|
|fS c c}}w c c}}w c c}}w )zBuild word-to-index vocabulary.   c                     | d    S NrH   r'   xs    r)   <lambda>z"build_vocabulary.<locals>.<lambda>b   s    adU r(   keyNu   📚 Vocabulary size: )r	   intsorteditems	enumerater1   r;   )r=   max_vocab_sizeword_countswordsorted_wordsrC   cvocab_wordsidxword2idxidx2words              r)   build_vocabularyr]   \   s    c"K DQ +++-?CL!-o~!>?A1?K?+4[+ABic4c	BHB+3>>+;<idCT	<H<	"3{#3"4
56X{** @B<s   B4%B:C c                    t        j                  d||fd      }t        j                  |      }	 t	        j
                  d|      }|j                  d| |      }|d   |d	   z
  }|d
   |d   z
  }||z
  dz  }	||z
  dz  }
|j                  |	|
f| d	|       t        j                  |t        j                        dz  }d|z
  }|S #  	 t	        j
                  d|      }n#  t	        j                         }Y nxY wY xY w)z)Render a word as a grayscale 28x28 image.L   )colorz//usr/share/fonts/truetype/dejavu/DejaVuSans.ttfz	arial.ttf)r   r   )font   r      rH   )fillrb   dtypeg     o@      ?)r   newr   Drawr   truetypeload_defaulttextbboxtextnparrayfloat32)rV   sizer   imgdrawrb   bbox
text_widthtext_heightrL   y	img_arrays               r)   render_word_imagerz   o   s	   
))C$S
1C>>#D,!!"SU^_ ==D=1Da47"Jq'DG#K	
	q A		!AIIq!fdI.BJJ/%7IiI%,	,%%k9=D	,))+Ds#   C C>
C! C>!C97C>c                   $    e Zd ZdZddZd Zd Zy)WordDatasetz3Dataset with word images + skip-gram context pairs.c                    g | _         || _        t        |      | _        t	        |j                               }|D cg c]	  }||v s| }}t        |      |kD  r|d | }|| _        |D cg c]  }||   	 c}| _        t        |      D ]  \  }}	||	   }
t        d||z
        }t        t        |      ||z   dz         }g }t        ||      D ]  }||k7  s	|j                  |||             ! g }t        t        j                        D ]B  }t        j                   t#        |j                                     }|j                  ||          D 	 t%        |	      }| j                   j                  ||
||d        t-        dt        | j                          d       y c c}w c c}w #  t'        j(                  dt&        j*                        }Y xxY w)Nr   rH   )r   r   rf   )imageword_idxcontext_indicesneg_indicesu   🖼️ Dataset size: z samples)samplesr[   r;   r   setkeysr=   word2idx_listrS   maxminrangeappendcfgr   randomchoicelistrz   ro   zerosrq   r1   )selfr=   r[   context_windowmax_samples	vocab_setrC   valid_wordsirV   rZ   startendr   jr   _neg_wordrs   s                      r)   __init__zWordDataset.__init__   s    x=(	"':Q1	>q::{k)%l{3K !
3>?ahqk? !- 	GAt4.C 1~-.Ec+&N(:Q(>?C O5#& E6#**8KN+CDE
 K3,,- 7!==hmmo)>?""8H#567;'- LL#2*	! /	< 	&s4<<'8&9BCQ ; @.;hhxrzz:s   	FF*F$F))'Gc                 ,    t        | j                        S N)r;   r   )r   s    r)   __len__zWordDataset.__len__   s    4<<  r(   c                     | j                   |   }t        j                  |d         j                  d      t        j                  |d   g      t        j                  |d         t        j                  |d         fS )Nr~   r   r   r   r   )r   r$   FloatTensor	unsqueeze
LongTensor)r   rZ   samples      r)   __getitem__zWordDataset.__getitem__   st    c"fWo.88;fZ012V$567VM23	
 	
r(   N)rc   iP  )r   r   r   __doc__r   r   r   r'   r(   r)   r|   r|      s    =.D`!
r(   r|   c                    t        |  \  }}}}t        j                  |d      }t        j                  |d      }t        d |D              rt	        |dd      }n0t        j
                  t        |       dft        j                        }t        d |D              rt	        |dd      }n0t        j
                  t        |       dft        j                        }||||fS )	z/Pad variable-length context and negative lists.r   dimc              3   B   K   | ]  }|j                         d kD    ywr   Nnumel.0ts     r)   	<genexpr>z%collate_word_batch.<locals>.<genexpr>   s     
.Q1779q=
.   T)batch_firstpadding_valuerf   c              3   B   K   | ]  }|j                         d kD    ywr   r   r   s     r)   r   z%collate_word_batch.<locals>.<genexpr>   s     
*Q1779q=
*r   )zipr$   stackanyr   emptyr;   long)batchimagesr   context_idxneg_idxs        r)   collate_word_batchr      s    -0%[*FHk7[[Q'F{{8+H

.+
..";DPRSkk3u:q/D

*'
**wDK++s5z1oUZZ@8['11r(   c                   ,     e Zd ZdZd fd	ZddZ xZS )WordMLPz
    Conv2D + Dense model with two heads:
    1. Classification head (word identity)
    2. Skip-gram context head (predict context words)
    c                    t         |           t        j                  t        j                  dddd      t        j
                         t        j                  d      t        j                  dddd      t        j
                         t        j                  d      t        j                  dddd      t        j
                         t        j                  d      	      | _        t        j                         | _	        t        j                  d	|      | _        t        j                  ||      | _        t        j                  t        j                  ||      t        j
                         t        j                  ||            | _        t        j                  ||      | _        t        j                   j#                  | j                  j$                  d
|z  d|z         y )NrH       rd   )kernel_sizepaddingrc   )r   r   r   i  g      r   )superr   nn
SequentialConv2dReLU	MaxPool2dfeaturesFlattenflattenLinear	embedding
classifiercontext_predictor	Embeddingword_embeddingsinituniform_weight)r   r   r   	__class__s      r)   r   zWordMLP.__init__   sI    IIaA6GGILLQ' IIb"!Q7GGILLQ' IIb#1a8GGILLQ'
" zz| ;	: ))I{;
 "$IIi+GGIIIi+"
  "||KC 	--44d9nc)mTr(   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|r| j
                  j                  }||||fS |||fS r   )r   r   r   r   r   r   r   )r   rL   return_context_embedsembed
cls_logitscontext_predword_embedss          r)   forwardzWordMLP.forward  s    MM!LLO q! __U+
 --e4 ..55KulK??5,..r(   )r   )F)r   r   r   r   r   r   __classcell__)r   s   @r)   r   r      s    +UZ/r(   r   c                    | j                  d      }| j                  d      }| j                  }t        j                  | d      }	t        j                  |d      }
t        j                  |j                  d      }g }g }t        |      D ]  }|	|   }||   D ]9  }t        |j                               }|dk\  s"||   }|j                  ||f       ; ||   D ]9  }t        |j                               }|dk\  s"||   }|j                  ||f       ;  t        |      dk(  rt        j                  d|      S d}|D ]J  \  }}t        j                  ||      }|t        j                  t        j                  |      dz          z  }L d}|D ]K  \  }}t        j                  ||      }|t        j                  t        j                  |       dz          z  }M ||z   t        t        |      t        |      z   d      z  }|S )z
    Skip-gram loss with negative sampling.
    
    Maximize: sim(target, context_word) for positive pairs
    Minimize: sim(target, random_word) for negative pairs
    
    Uses in-batch negatives for efficiency.
    r   rH   r           r%   :0yE>)rr   r%   F	normalizer   r   rP   itemr   r;   r$   tensordotlogsigmoidr   )
embeddingsr   r   r   r   r   r   r   r%   targetcontext_pred_normr   	pos_pairs	neg_pairsr   
target_vecctx_idxctx_vecr   neg_vecpos_losstgtctxscoreneg_lossneg
total_losss                              r)   skipgram_lossr   2  s
    #J"IF [[+FLa8 ++o44!<K II: 8AY
 'q) 	8G',,.)G!|%g.  *g!67		8 #1~ 	8G',,.)G!|%g.  *g!67		88" 9~||C// H <S		#s#UYYu}}U3d:;;;<
 H =S		#s#UYYu}}eV4t;<<<= X%S^c)n-La)PPJr(   c           
         | j                  d      }| j                  d      }| j                  }t        j                  | d      }t        j                  |d      }t        j                  |j                  d      }	|	|   }
t        j                  |
d      }
t        j                  ||z  d      }t        j                  t        j                  |      dz         j                          }d}d}t        |      D ]  }||   }||   j                         }g }t        |      D ]*  }||   j                         |k7  s|j                  |       , t        |      dkD  sdt        j                  |t!        |t        |                  }|D ]K  }||   }t        j"                  ||      }|t        j                  t        j                  |      dz         z  }M  |dkD  r| ||z  z  }||z   S t        j$                  d|      }||z   S )z
    In-batch skip-gram loss with efficient negative sampling.
    
    For each target, the other words in the batch (with different labels)
    serve as negative samples.
    r   rH   r   r   r   r   r   )rr   r%   r   r   r   r$   sumr   r   meanr   r   r   r;   r   r   r   r   r   )r   r   r   word_idx_batchr   r   r   r%   r   r   context_embeds
pos_scoresr   r   r   r   r   labelr   r   r   r   s                         r)   inbatch_skipgram_lossr  o  s    #J"IF [[+F;;|3L ++o44!<K !0N[[Q7N 6L0a8J		%--
3d:;@@BBH HM: CQiq!&&( z" 	&Aa %%'50""1%	& {a --SKHX5YZK  C )		#w/EIIemmE&:T&ABBCC& A~9
] :; h <<F3hr(   c                    t         j                  j                  | j                         |      }t	        j
                         }| j                  t        j                         t        |      D ]t  }| j                          d}d}	d}
d}d}t        |      D ]  \  }\  }}}}|j                  t        j                        }|j                  d      j                  t        j                        }|j                  t        j                        }|j                  t        j                        }|j                           | |      \  }}} |||      }t        ||| j                  |||      }t        j                   |z  t        j"                  |z  z   }|j%                          t         j                  j&                  j)                  | j                         d       |j+                          ||j-                         z  }|	|j-                         z  }	|
|j-                         z  }
|j/                  d      }|||k(  j1                         j-                         z  }||j3                  d      z  } d	|z  |z  }|t5        |      z  }|	t5        |      z  }|
t5        |      z  }| j7                          d}d}d}t        j8                         5  |D ]  \  }}}}|j                  t        j                        }|j                  d      j                  t        j                        } | |      \  }} }  |||      }||j-                         z  }|j/                  d      }|||k(  j1                         j-                         z  }||j3                  d      z  } 	 d
d
d
       d	|z  |z  }!|t5        |      z  }"t;        d|dz    d| d|dd|dd|dd|dd|!dd       w | S # 1 sw Y   MxY w)z-Train with classification + skip-gram losses.)r!   r   r   r   rh   )max_normrH   r   d   NzEpoch /z	 | Loss: .4fz (Cls: z + SG: z) | Train: .2fz	% | Val: %)r$   optimAdam
parametersr   CrossEntropyLosstor   r%   r   trainrS   squeeze	zero_gradr   r   r"   r#   backwardutilsclip_grad_norm_stepr   argmaxr   rr   r;   evalno_gradr1   )#modeltrain_loader
val_loaderr    r!   	optimizercls_criterionepoch
train_losstrain_cls_losstrain_sg_losstrain_correcttrain_total	batch_idxr   r   r   r   r   r   r   cls_losssg_losslosspreds	train_accavg_train_lossavg_cls_lossavg_sg_lossval_lossval_correct	val_totalr   val_accavg_val_losss#                                      r)   train_modelr3    s      !1!1!3 ;I'')M	HHSZZv N@
CL\CZ %	,?I?+wYYszz*F''+..szz:H%..4Kjj,G! 49=0J
L %Z:H $%%G >>H,s/B/BW/LLDMMOHHNN**5+;+;+=*LNN$))+%Jhmmo-NW\\^+M%%!%,Eex/446;;==M8==++KK%	,N -'+5	#c,&77%L(99#c,&77 	

	]]_ 	.:D .6+w3::.#++B/223::>#(= 
Aq$Z:DIIK'"))a)0 1668==??X]]1--	.	. #i/#j/1uQwiq )%c*',s1C7;WZJ[ \!#i}A? 	@YN@` L+	. 	.s   CO((O1	c                    | j                          t        j                         }d}d}d}t        t              }t        t              }	t        j                         5  |D ]3  \  }
}}}|
j                  t        j                        }
|j                  d      j                  t        j                        } | |
      \  }}} |||      }||j                         z  }|j                  d      }|||k(  j                         j                         z  }||j                  d      z  }t        |j!                         |j!                               D ]A  \  }}|	|j                         xx   dz  cc<   ||k(  s'||j                         xx   dz  cc<   C 6 	 ddd       d|z  |z  }|t#        |      z  }t%        d       t%        d	       t%        d
       t%        d|d       t%        d|dd| d| d       |	D ci c]  }|||   t'        |	|   d      z   }}t)        |j+                         d       }t%        d       |dd D ]1  \  }}|j-                  |d| d      }t%        d| d|dz  dd       3 t%        d       |dd D ]1  \  }}|j-                  |d| d      }t%        d| d|dz  dd       3 t%        d       t%        d       | j                          t        j                         5  g d g d!g d"g d#g d$d%}|j+                         D ]a  \  }}|D cg c]	  }||v s| } }t#        |       d&k  r*t%        d'| d(|         g }!| D ]  }t        j.                  t1        |            j3                  d      j3                  d      j                  t        j                        }" | |"      \  }}#}|!j5                  t7        j8                  |#d      j                         j!                         j;                                 t=        |       D ][  \  }$}%t=        |       D ]H  \  }&}'|&|$kD  st?        j@                  |!|$   |!|&         }(|(d)kD  rd*nd+})t%        d,|) d-|% d.|' d/|(d0       J ] d 	 ddd       t%        d1       g d2}|D ]s  }||vr	t        j.                  t1        |            j3                  d      j3                  d      j                  t        j                        }" | |"      \  }}#}t7        j8                  |#d      }# | |"      \  }}}*t7        j8                  |*d      }*t7        j8                  | jB                  jD                  d      }+t        jF                  |*|+jH                        j                         },d}-|,jK                  |-dz         \  }}.g }/|.D ]?  }|j                         }|||   k7  s|/j5                  |j-                  |d3|              A t%        d4| d5|/d|-         v t%        d
       |S # 1 sw Y   xY wc c}w c c}w # 1 sw Y   xY w)6z0Test and analyze learned semantic relationships.r   r   r   rH   r   Nr  z=
============================================================u   📊 TEST RESULTS<============================================================zTest Loss: r  zTest Accuracy: r	  z% (r  )c                     | d    S rJ   r'   rK   s    r)   rM   ztest_model.<locals>.<lambda>4  s    !A$ r(   rN   u   
📈 Top 5:r   z<idx:>z  z: z.1fr
  u   
📉 Bottom 5:u"   
🔗 SKIP-GRAM SEMANTIC CLUSTERS:zCTesting if words appearing in similar contexts cluster together...
)theaan)andorbut)inonatr  from)isarewaswerebebeen)r=  r:  rD  r  of)articlesconjunctionsprepositionsverbs_becommonrc   z  [z]: r   u   ✅u   ⚠️z    z 'z' <-> 'z': z.3fu4   
🔍 Top Similar Words (Skip-Gram Embedding Space):)r=  r:  rD  rJ  r  zidx:z  'u   ' → )&r  r   r  r	   rP   r$   r  r  r   r%   r  r   r  r   rr   r   r   r;   r1   r   rQ   rR   getr   rz   r   r   r   r   numpyrS   ro   r   r   r   mmTtopk)0r  test_loaderr[   r\   	criterion	test_losstest_correct
test_totalclass_correctclass_totalr   r   r   r   r   r   r(  r)  predtruetest_accavg_test_lossk	class_acc
sorted_accrZ   accrV   test_groups
group_name
test_wordsrC   r   embedsrs   r   r   w1r   w2simstatusr   r   similaritiestop_ktop_indicessimilar_wordss0                                                   r)   
test_modelrp  	  s*    
JJL##%IILJ$Mc"K	 46A 	42FHk7YYszz*F''+..szz:H$V}J1Z2D$I%%!%,EUh.335::<<L(--**J!%))+x||~> 4
dDIIK(A-(4<!$))+.!3.4	44& \!J.HK 00M	-	
	&M	Kc*
+,	OHS>\N!J<q
IJ EPPqM!$SQ%:::PIP	)?J	/rN +S||C5Q04&3s73-q)*+ 

rsO +S||C5Q04&3s73-q)*+
 

/0	
PQ	JJL	 L +0<B6
 '2&7&7&9 	L"J
&0BAM1BKB;!#C
|3{m45 F# Q''(9$(?@JJ1MWWXYZ]]^a^h^hi#Cj5!akk%Q7??AEEGMMOPQ #;/ L2&{3 LEAr1u ffVAYq	:*-)VHBrd'"SS	JK	LL	LLB 

AB1J 9x 1$ 78BB1EOOPQRUUVYV`V`aCj5!Eq) #3Z1l{{<Q7 kk%"7"7">">AFxxkmm<DDF %**5195; 	FC((*Chtn$$$X\\#cU|%DE	F
 	D6fu 5678598 
&MOc4 4: Q: CL LsE   DX#7 X# X0=.X:+	X55X59DX:AX:#X-5X::Yc            
         t        d       t        d       t        d       t        t        j                        } t        d       t	        | d      \  }}}t        |      t        _        t        d       t        | |dd	
      }t        d       t        |      }t        d|z        }t        d|z        }||z
  |z
  }t        j                  j                  j                  ||||g      \  }	}
}t        |	t        j                  dt              }t        |
t        j                  dt              }t        |t        j                  dt              }t        dt        |	       dt        |
       dt        |              t        d       t!        t        j                  t        j"                        }t        |       t        d       t%        |||t        j&                  t        j(                        }t        d       t+        ||||      }t        j,                  |j/                         ||t        j                  t        j"                  ddd       t        d       t        d|dd       y )Nu)   🧠 Word-MLP with Skip-Gram Context Headr5  u'   
📂 Step 1: Loading markdown files...u$   
📚 Step 2: Building vocabulary...i  )rT   u$   
🖼️ Step 3: Creating dataset...rc   i0u  )r   r   u   
📊 Step 4: Splitting data...gffffff?g333333?T)r   shuffle
collate_fnFz	  Train: z | Val: z	 | Test: u"   
🏗️ Step 5: Building model...)r   r   u6   
🚀 Step 6: Training (Classification + Skip-Gram)...)r    r!   u   
🧪 Step 7: Testing...)model_state_dictr[   r\   configzword_mlp_skipgram.ptu+   
💾 Model saved to 'word_mlp_skipgram.pt'zFinal Test Accuracy: r	  r
  )r1   rD   r   r   r]   r;   r   r|   rP   r$   r  datarandom_splitr   r   r   r   r   r3  r    r!   rp  save
state_dict)r=   r[   r\   rY   datasettotal
train_sizeval_size	test_sizetrain_datasetval_datasettest_datasetr  r  rU  r  r^  s                    r)   mainr    s   	
56	&M 

450E 

12&6uT&R#Hh+&CO 

12%!OG 

,-LES5[!J4%< H
"X-I/4{{/?/?/L/L*h	20,M; mPTastLKCNNE^pqJ\cnne`rsK	Ic-()#k2B1C9SQ]M^L_
`a 

/03==IE	%L 

CD|Z

svvVE 

%&%hAH 
JJ!,,.??
	  

89	!(3q
12r(   __main__)i  )r   r   )r   r   ))r   osr7   r   rQ  ro   r$   torch.nnr   torch.nn.functional
functionalr   torch.utils.datar   r   torch.nn.utils.rnnr   PILr   r   r   collectionsr	   pathlibr
   r   r   rD   r2   r]   rz   r|   r   Moduler   r   r  r3  rp  r  r   r'   r(   r)   <module>r     s   
 
 	       0 + + + # J J2 h0<+&>=
' =
@2.G/bii G/Z:z6xX|~H;3z zF r(   