
    V iZ                     P   d Z ddlZddlZddlZddlZddlZddlmZ ddl	mc m
Z ddlmZmZ ddlmZ ddlmZmZmZ ddlmZ ddlmZ  G d d	      Z e       Zd
 Zd ZddZddZ G d de      Zd Z  G d dejB                        Z"ddZ#ddZ$d Z%d Z&e'dk(  r e&        yy)u   
Word-MLP: Conv2D + Dense for Visual Word Embeddings
Loads markdown files → Renders words as 28x28 images → Trains → Tests
    N)Dataset
DataLoader)pad_sequence)Image	ImageDraw	ImageFont)defaultdict)Pathc                       e Zd ZdZdZdZdZdZdZdZ	dZ
 ej                  ej                  j                         rd      Zyd	      Zy)
Configz./books         N@   MbP?cudacpu)__name__
__module____qualname__book_folderimg_size	font_size	embed_dimnum_classes
batch_sizeepochslrtorchdevicer   is_available     ex01.pyr   r      sX    K HI IK JF	B U\\EJJ$;$;$=&IF5IFr#   r   c                 D   g }t        |       }|j                         st        d|  d       t               S |j	                  d      D ]  }t        d|j
                          t        |ddd      5 }|j                         }d	d	d	       t        j                  d
      }|j                  |D cg c]  }|j                          c}        t        |      dk(  rt        d|  d       t               S t        dt        |       d       |S # 1 sw Y   xY wc c}w )z1Load all .md files from folder and extract words.u   ⚠️ Folder 'z' not found. Using sample text.z*.mdu   📖 Loading: rzutf-8ignore)encodingerrorsN[a-zA-Z]{2,}r   u   ⚠️ No words found in z. Using sample text.u   ✅ Extracted z words from markdown files.)r
   existsprintsample_wordsglobnameopenreadrefindallextendlowerlen)folder_pathwordsfoldermd_filefcontent	extractedws           r$   load_markdown_filesr?   1   s   E+F==?},KLM~;;v& 5w||n-.'3B 	affhG	 JJ8	3Aaggi345 5zQ)+6JKL~	N3u:,&A
BCL	 	
 4s   2D0D
D	c                  N    d} t        j                  d| j                               S )z1Fallback sample words if no markdown files found.a	  
    intelligence artificial machine learning neural network deep 
    consciousness awareness thinking reasoning planning algorithm 
    data information knowledge wisdom understanding insight logic 
    mathematics physics chemistry biology science research study
    theory hypothesis experiment observation analysis synthesis
    computer digital electronic computational programming code
    network connection relationship system structure function
    energy power strength force motion dynamics change growth
    r*   )r2   r3   r5   )sample_texts    r$   r-   r-   J   s%    	K ::o{'8'8':;;r#   c                    t        t              }| D ]  }||xx   dz  cc<    t        |j                         d       }|d| D cg c]  \  }}|	 }}}t	        |      D ci c]  \  }}||
 }	}}|	j                         D ci c]  \  }}||
 }
}}t        dt        |              |	|
|fS c c}}w c c}}w c c}}w )zBuild word-to-index vocabulary.   c                     | d    S NrC   r"   xs    r$   <lambda>z"build_vocabulary.<locals>.<lambda>c   s    adU r#   keyNu   📚 Vocabulary size: )r	   intsorteditems	enumerater,   r6   )r8   max_vocab_sizeword_countswordsorted_wordsr>   cvocab_wordsidxword2idxidx2words              r$   build_vocabularyrX   \   s    c"K DQ +++-?CL ".o~!>?A1?K?+4[+ABic4c	BHB+3>>+;<idCT	<H<	"3{#3"4
56X{** @B<s   B4%B:C c                    t        j                  d||fd      }t        j                  |      }	 t	        j
                  d|      }|j                  d| |      }|d   |d	   z
  }|d
   |d   z
  }||z
  dz  }	||z
  dz  }
|j                  |	|
f| d	|       t        j                  |t        j                        dz  }d|z
  }|S #  	 t	        j
                  d|      }n#  t	        j                         }Y nxY wY xY w)z)Render a word as a grayscale 28x28 image.L   )colorz//usr/share/fonts/truetype/dejavu/DejaVuSans.ttfz	arial.ttf)r   r   )font   r      rC   )fillr]   dtypeg     o@      ?)r   newr   Drawr   truetypeload_defaulttextbboxtextnparrayfloat32)rQ   sizer   imgdrawr]   bbox
text_widthtext_heightrG   y	img_arrays               r$   render_word_imageru   r   s    ))C$S
1C>>#D,!!"SU^_ ==D=1Da47"Jq'DG#K	
	q A		!A 	IIq!fdI. BJJ/%7I iI/,	,%%k9=D	,))+Ds#   C C>
C! C>!C97C>c                   $    e Zd ZdZddZd Zd Zy)WordDatasetz2Dataset of word images with context co-occurrence.c           	      \   g | _         || _        t        |j                               }|D cg c]	  }||v s| }}t	        |      |kD  r|d | }t        |      D ]  \  }}	||	   }
t        d||z
        }t        t	        |      ||z   dz         }g }t        ||      D ]'  }||k7  s	||   |v s|j                  |||             ) g }t        t        dt	        |                  D ]B  }t        j                  t        |j                                     }|j                  ||          D 	 t        |	      }| j                   j                  ||
||d        t#        dt	        | j                          d       y c c}w #  t        j                  dt        j                         }Y sxY w)	Nr   rC      )r   r   ra   )imageword_idxcontext_indicesneg_indicesu   🖼️ Dataset size: z samples)samplesrV   setkeysr6   rN   maxminrangeappendrandomchoicelistru   rj   zerosrl   r,   )selfr8   rV   context_windowmax_samples	vocab_setr>   valid_wordsirQ   rU   startendr|   jr}   _neg_wordrn   s                      r$   __init__zWordDataset.__init__   s     (	"':Q1	>q:: {k)%l{3K !- 	GAt4.C 1~-.Ec+&N(:Q(>?C O5#& E6k!n8#**8KN+CDE
 K3q#o"678 7!==hmmo)>?""8H#567
;'- LL#2*	! 1	> 	&s4<<'8&9BCM ;8;hhxrzz:s   	E=E=-F'F+c                 ,    t        | j                        S N)r6   r~   )r   s    r$   __len__zWordDataset.__len__   s    4<<  r#   c                     | j                   |   }t        j                  |d         j                  d      t        j                  |d   g      t        j                  |d         t        j                  |d         fS )Nrz   r   r{   r|   r}   )r~   r   FloatTensor	unsqueeze
LongTensor)r   rU   samples      r$   __getitem__zWordDataset.__getitem__   st    c"fWo.88;fZ012V$567VM23	
 	
r#   N)r^   iP  )r   r   r   __doc__r   r   r   r"   r#   r$   rw   rw      s    <,D\!
r#   rw   c                    t        |  \  }}}}t        j                  |d      }t        j                  |d      }t        d |D              rt	        |dd      }n0t        j
                  t        |       dft        j                        }t        d |D              rt	        |dd      }n0t        j
                  t        |       dft        j                        }||||fS )	z>Pad variable-length context and negative lists within a batch.r   dimc              3   B   K   | ]  }|j                         d kD    ywr   Nnumel.0ts     r$   	<genexpr>z%collate_word_batch.<locals>.<genexpr>   s     
.Q1779q=
.   T)batch_firstpadding_valuera   c              3   B   K   | ]  }|j                         d kD    ywr   r   r   s     r$   r   z%collate_word_batch.<locals>.<genexpr>   s     
*Q1779q=
*r   )zipr   stackanyr   emptyr6   long)batchimagesr{   context_idxneg_idxs        r$   collate_word_batchr      s    -0%[*FHk7[[Q'F{{8+H

.+
..";DPRSkk3u:q/D

*'
**wDK++s5z1oUZZ@8['11r#   c                   *     e Zd ZdZd fd	Zd Z xZS )WordMLPz)Conv2D + Dense model for word embeddings.c                 r   t         |           t        j                  t        j                  dddd      t        j
                         t        j                  d      t        j                  dddd      t        j
                         t        j                  d      t        j                  dddd      t        j
                         t        j                  d      	      | _        t        j                         | _	        t        j                  d	|      | _        t        j                  ||      | _        y )
NrC       r_   )kernel_sizepaddingr^   )r   r   r   i  )superr   nn
SequentialConv2dReLU	MaxPool2dfeaturesFlattenflattenLinear	embedding
classifier)r   r   r   	__class__s      r$   r   zWordMLP.__init__   s     IIaA6GGILLQ' IIb"!Q7GGILLQ' IIb#1a8GGILLQ'
$ zz| ;	: ))I{;r#   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }||fS r   )r   r   r   r   )r   rG   embedlogitss       r$   forwardzWordMLP.forward  sG    MM!LLO q! 'u}r#   )r   )r   r   r   r   r   r   __classcell__)r   s   @r$   r   r      s    3<<r#   r   c                 
   t         j                  j                  | j                         |      }t	        j
                         }| j                  t        j                         t        |      D ]  }| j                          d}d}	d}
t        |      D ]  \  }\  }}}}|j                  t        j                        }|j                  d      j                  t        j                        }|j                           | |      \  }} |||      }|j                  d   dkD  rIt        |||j                  t        j                        |j                  t        j                              }n2t        j                   d      j                  t        j                        }|d|z  z   }|j#                          |j%                          ||j'                         z  }|j)                  d      }|	||k(  j+                         j'                         z  }	|
|j-                  d      z  }
 d|	z  |
z  }|t/        |      z  }| j1                          d}d}d}t        j2                         5  |D ]  \  }}}}|j                  t        j                        }|j                  d      j                  t        j                        } | |      \  }} |||      }||j'                         z  }|j)                  d      }|||k(  j+                         j'                         z  }||j-                  d      z  } 	 d	d	d	       d|z  |z  }|t/        |      z  }t5        d
|dz    d| d|dd|dd|dd|dd        | S # 1 sw Y   IxY w)z3Train the model with classification + context loss.)r           r   r   rC   g?r   d   NzEpoch /z | Train Loss: .4fz Acc: .2fz% | Val Loss: %)r   optimAdam
parametersr   CrossEntropyLosstocfgr    r   trainrN   squeeze	zero_gradshapecompute_context_losstensorbackwardstepitemargmaxsumrm   r6   evalno_gradr,   )modeltrain_loader
val_loaderr   r   	optimizer	criterionepoch
train_losstrain_correcttrain_total	batch_idxr   r{   r   r   r   
embeddingscls_losscontext_losslosspreds	train_accavg_train_lossval_lossval_correct	val_totalval_accavg_val_losss                                r$   train_modelr    sL      !1!1!3 ;I##%I	HHSZZv ?C
CL\CZ 	,?I?+wYYszz*F''+..szz:H!!&vFJ !2H   #a'3J+..Y\YcYcJdfmfpfpqtq{q{f|}$||C033CJJ? cL00DMMONN$))+%JMMaM(Eex/446;;==M8==++K5	,8 -'+5	#c,&77 	

	]]_ 	.:D .6+w3::.#++B/223::>%*6]"
 2DIIK'!, 1668==??X]]1--	.	. #i/#j/1uQwiq )+C0yo F',F73-qB 	C{?CB L+	. 	.s   %CM99N	c                 N   | j                  d      }t        j                  | d      } | j                  }t	        j
                  d|      }t        t              }t        |j                               D ]"  \  }	}
|t        |
         j                  |	       $ t        |      D ]p  }	| |	   }t	        j
                  d|      }||	   D ]  }t        |j                               }|dk  r"|j                  |g       D ]P  }||	k(  r	|t        j                  |j!                  d      | |   j!                  d            j#                         z   }R  t	        j
                  d|      }||	   D ]  }t        |j                               }|dk  r"|j                  |g       D ]f  }||	k(  r	|t        j$                  |t        j                  |j!                  d      | |   j!                  d            j#                         z
        z   }h  |||z   z  }s |t'        |d      z  S )zS
    Contrastive loss over in-batch matches for context and negative word ids.
    r   rC   r   r   )r    )rm   F	normalizer    r   r   r	   r   rN   tolistrK   r   r   r   getpairwise_distancer   r   relur   )r   r{   r   r   marginr   r    
total_losslabel_to_positionsr   labelembpos_lossctx_idx	ctx_labelr   neg_loss	neg_labels                     r$   r   r   j  s     #J ZQ/JFc&1J$T*hoo/0 153u:&--a01 : !*m <<F3"1~ 
	GGLLN+I1}'++Ir: 6#a&9&9MM!$qM++A.' ')	
	 <<F3  	IINN,-I1}'++Ir: 6#affQ00a("1//2 gi ' 		 	h))
C!*F J***r#   c                 :   | j                          t        j                         }d}d}d}t        t              }t        t              }	t        j                         5  |D ]2  \  }
}}}|
j                  t        j                        }
|j                  d      j                  t        j                        } | |
      \  }} |||      }||j                         z  }|j                  d      }|||k(  j                         j                         z  }||j                  d      z  }t        |j!                         |j!                               D ]A  \  }}|	|j                         xx   dz  cc<   ||k(  s'||j                         xx   dz  cc<   C 5 	 ddd       d|z  |z  }|t#        |      z  }t%        d       t%        d	       t%        d
       t%        d|d       t%        d|dd| d| d       t%        d       |	D ci c]  }|||   t'        |	|   d      z   }}t)        |j+                         d       }t%        d       |dd D ]=  \  }}|j-                  |d| d      }t%        d| d|dz  dd||    d|	|    d	       ? t%        d       |dd D ]=  \  }}|j-                  |d| d      }t%        d| d|dz  dd||    d|	|    d	       ? t%        d       | j                          d}t        j                         5  |D ]  \  }
}}}t/        t1        d|
j                  d                  D ]  }|
||dz    j                  t        j                        }||   j                         } | |      \  }} |j                  d      j                         }!|j-                  |d| d      }"|j-                  |!d|! d      }#|!|k(  rdnd }$t%        d!|$ d"|" d#|# d$       |dz  }|d%k\  s n |d%k\  s n ddd       t%        d&       g d'}%|%D &cg c]	  }&|&|v s|& }'}&t#        |'      d(k\  r| j                          t        j                         5  |'dd) D ]]  }t        j2                  t5        |            j7                  d      j7                  d      j                  t        j                        } | |      \  } }(g })|j+                         D ]  \  }&}|&|k7  st        j2                  t5        |&            j7                  d      j7                  d      j                  t        j                        }* | |*      \  } }+t9        j:                  |(|+      j                         },|)j=                  |&|,f        |)j?                  d*        |)dd+ D &-cg c]  \  }&}-|&	 }.}&}-t%        d,| d-|.        ` 	 ddd       t%        d
       |S # 1 sw Y   RxY wc c}w # 1 sw Y   xY wc c}&w c c}-}&w # 1 sw Y   @xY w).z,Test the model and show qualitative results.r   r   r   rC   r   Nr   z=
============================================================u   📊 TEST RESULTS<============================================================zTest Loss: r   zTest Accuracy: r   z% (r   )u   
📈 Per-Class Accuracy:c                     | d    S rE   r"   rF   s    r$   rH   ztest_model.<locals>.<lambda>  s    !A$ r#   rI   z  Top 5:ry   z<idx:>z    z: z.1fz  Bottom 5:u   
🔍 Sample Predictions:u   ✅u   ❌z  z True: 'z' | Pred: ''
   u8   
🔗 Embedding Similarity (semantically similar words):)theandisarewaswerer^      c                     | d    S rE   r"   rF   s    r$   rH   ztest_model.<locals>.<lambda>
  s    adU r#   r_   z  'u   ' → ) r   r   r   r	   rK   r   r   r   r   r    r   r   r   r   rm   r   r   r6   r,   r   rL   rM   r  r   r   r   ru   r   r  cosine_similarityr   sort)/r   test_loaderrV   rW   r   	test_losstest_correct
test_totalclass_correctclass_totalr   r{   r   r   r   r   r   r   predtruetest_accavg_test_lossk	class_acc
sorted_accrU   accrQ   sample_countr   rn   true_idxr   pred_idx	true_word	pred_wordstatus
test_wordsr>   available_wordsr  	distancesimg2emb2distdsimilars/                                                  r$   
test_modelrA    s    
JJL##%IILJ  $Mc"K	 46A 	42FHk7YYszz*F''+..szz:H!&vFJVX.D$IMMaM(EUh.335::<<L(--**J "%))+x||~> 4
dDIIK(A-(4<!$))+.!3.4	44( \!J.HK 00M	-	
	&M	Kc*
+,	OHS>\N!J<q
IJ 

&'DOPqM!$SQ%:::PIP	)?J	*rN WS||C5Q0TF"SWSM]3-?,@+cBRASSTUVW 
-rsO WS||C5Q0TF"SWSM]3-?,@+cBRASSTUVW
 

&'	JJLL	 6A 	2FHk73q&++a.12 Qqsm&&szz2#A;++-!#J	!==Q=/446$LLU8*A3FG	$LLU8*A3FG	"*h"6E6((9+[1MN!2% r!#	* 

EF;J",>QXq>O>
?q 

]]_ 	3'+ 3''(9$(?@JJ1MWWXYZ]]^a^h^his3 	&nn. 4FAsDy$001B11EFPPQRS]]^_`ccdgdndno"'+4 223=BBD!((!T34 ?3)22A7A177D6y123	3$ 
&MOA4 4> Q$ . ?& 8	3 	3sd   DW'6 W'
W4C)W9 W9	W9'	X1X*BX6B8X.X
:X'W19XXXc            
         t        d       t        d       t        d       t        t        j                        } t        d       t	        | d      \  }}}t        |      t        _        t        d       t        | |dd	
      }t        d       t        |      }t        d|z        }t        d|z        }||z
  |z
  }t        j                  j                  j                  ||||g      \  }	}
}t        |	t        j                  dt              }t        |
t        j                  dt              }t        |t        j                  dt              }t        dt        |	       dt        |
       dt        |              t        d       t!        t        j                  t        j"                        }t        |       t        d       t%        |||t        j&                  t        j(                        }t        d       t+        ||||      }t        j,                  |j/                         ||t        j                  t        j"                  ddd       t        d       t        d|dd       y )Nu-   🧠 Word-MLP: Conv2D + Dense Word Embeddingsr  u'   
📂 Step 1: Loading markdown files...u$   
📚 Step 2: Building vocabulary...i  )rO   u$   
🖼️ Step 3: Creating dataset...r^   i0u  )r   r   u   
📊 Step 4: Splitting data...gffffff?g333333?T)r   shuffle
collate_fnFz	  Train: z | Val: z	 | Test: u"   
🏗️ Step 5: Building model...)r   r   u   
🚀 Step 6: Training...)r   r   u   
🧪 Step 7: Testing...)model_state_dictrV   rW   configzword_mlp_model.ptu(   
💾 Model saved to 'word_mlp_model.pt'zFinal Test Accuracy: r   r   )r,   r?   r   r   rX   r6   r   rw   rK   r   utilsdatarandom_splitr   r   r   r   r   r  r   r   rA  save
state_dict)r8   rV   rW   rT   datasettotal
train_sizeval_size	test_sizetrain_datasetval_datasettest_datasetr   r   r%  r   r-  s                    r$   mainrT    s   	
9:	&M 

450E 

12&6uT&R#Hh+&CO 

12%!OG 

,-LES5[!J4%< H
"X-I/4{{/?/?/L/L*h	20,M; mPTastLKCNNE^pqJ\cnne`rsK	Ic-()#k2B1C9SQ]M^L_
`a 

/03==IE	%L 

&'|Z

svvVE 

%&%hAH 
JJ!,,.??
	  

56	!(3q
12r#   __main__)i  )r   r   )r   r   )rc   )(r   osr2   r   numpyrj   r   torch.nnr   torch.nn.functional
functionalr  torch.utils.datar   r   torch.nn.utils.rnnr   PILr   r   r   collectionsr	   pathlibr
   r   r   r?   r-   rX   ru   rw   r   Moduler   r  r   rA  rT  r   r"   r#   r$   <module>ra     s   
 
 	       0 + + + # J J( h2<$+, L;
' ;
|2.,bii ,dIV2+pnh;3z zF r#   