
    5iEK                     D   d Z ddlZddlZddlmZ ddlmZ ddlZddl	Z	ddl
mZ ddlmc mZ ddlmZ ddlmZmZ  G d d      Z e       Zd	 Zd
 ZddZd Z G d de      Zd Zd ZddZ G d dej<                        Zd Z d Z!ddZ"d Z#d Z$e%dk(  r e$        yy)a  
Next-word prediction with a two-stage reward schedule.

Goal:
- First guess from a word sequence
- If wrong, reveal the first letter of the target word and guess again
- Training should converge toward solving it on the first guess

Training strategy:
- Cross-entropy on both stages
- Exact-match reward shaping
- Hint reward decays over epochs so the model relies less on the hint
    N)defaultdict)Path)Categorical)Dataset
DataLoaderc                       e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
ZdZg dZ ej                   ej"                  j%                         rd      Zyd      Zy)Configz./books         N   @      MbP?皙??))        r   full_minus_one)333333?r   r   )      ?g?two_letters)g?ffffff?
one_letter)g?      ?r   cudacpu)__name__
__module____qualname__book_folder	embed_dim
hidden_dimhint_embed_dimnum_classesseq_len
batch_sizeepochslr	rl_weightbaseline_momentumhint_levelstorchdevicer   is_available     ex05.pyr	   r	      so    K IJNK G JF	BIK U\\EJJ$;$;$=&IF5IFr0   r	   c                 &   g }t        |       }|j                         st        d|  d       t               S |j	                  d      D ]  }t        d|j
                          t        |ddd      5 }|j                         }d d d        t        j                  d	      }|j                  |D cg c]  }|j                          c}        t        |      d
k(  r
t               S t        dt        |       d       |S # 1 sw Y   xY wc c}w )Nu   ⚠️ Folder 'z' not found. Using sample text.z*.mdu   📖 Loading: rzutf-8ignore)encodingerrors[a-zA-Z]{2,}r   u   ✅ Extracted z words from markdown files.)r   existsprintsample_wordsglobnameopenreadrefindallextendlowerlen)folder_pathwordsfoldermd_filefcontent	extractedws           r1   load_markdown_filesrL   <   s    E+F==?},KLM~;;v& 5w||n-.'3B 	affhG	 JJ8	3Aaggi345 5zQ~	N3u:,&A
BCL	 	 4s   2D0D
D	c                  N    d} t        j                  d| j                               S )Nz
    intelligence artificial machine learning neural network deep
    consciousness awareness thinking reasoning planning algorithm
    r7   )r?   r@   rB   )sample_texts    r1   r:   r:   S   s%    K ::o{'8'8':;;r0   c                    t        t              }| D ]  }||xx   dz  cc<    t        |j                         d       }|d | D cg c]  \  }}|	 }}}t	        |      D ci c]  \  }}||
 }	}}|	j                         D ci c]  \  }}||
 }
}}t        dt        |              |	|
|fS c c}}w c c}}w c c}}w )N   c                     | d    S )NrP   r/   )xs    r1   <lambda>z"build_vocabulary.<locals>.<lambda>`   s    adU r0   )keyu   📚 Vocabulary size: )r   intsorteditems	enumerater9   rC   )rE   max_vocab_sizeword_countswordsorted_wordsrK   _vocab_wordsidxword2idxidx2words              r1   build_vocabularyrb   [   s    c"K DQ +++-?CL!-o~!>?A1?K?+4[+ABic4c	BHB+3>>+;<idCT	<H<	"3{#3"4
56X{** @B<s   B4%B:C c                 r    | r| j                         syt        | j                               t        d      z
  S )N   a)isalphaordrB   )letters    r1   char_to_idxri   j   s+    )v||~S))r0   c                   $    e Zd ZdZddZd Zd Zy)SequenceHintDatasetzAContext sequence -> next word, plus target word for prefix hints.c                    g | _         || _        || _        t        |j	                               }|D cg c]  }||v st        |      dkD  s| }}t        |      |kD  r|d | }t        t        |      |z
        D ]I  }||||z    }	|||z      }
|	D cg c]  }||   	 }}||
   }| j                   j                  |||	|
d       K t        dt        | j                          d| d       y c c}w c c}w )NrP   )context_indices
target_idxcontext_wordstarget_wordu   🖼️ Sequence-hint dataset: z samples (seq_len=))	samplesr%   r`   setkeysrC   rangeappendr9   )selfrE   r`   r%   max_samples	vocab_setrK   valid_wordsiro   rp   rm   rn   s                r1   __init__zSequenceHintDataset.__init__s   s    (	"'IQ1	>c!fqjqII{k)%l{3Ks;''12 	A'AK8M%a'k2K4ABqx{BOB!+.JLL'6",%2#.		  	/DLL0A/BBTU\T]]^_`+ J Cs   	C.C.C.C3c                 ,    t        | j                        S N)rC   rr   )rw   s    r1   __len__zSequenceHintDataset.__len__   s    4<<  r0   c                     | j                   |   }t        j                  |d         t        j                  |d   g      |d   |d   fS )Nrm   rn   ro   rp   )rr   r,   
LongTensor)rw   r_   samples      r1   __getitem__zSequenceHintDataset.__getitem__   sT    c"V$567f\234?#=!	
 	
r0   N)r   iP  )r   r   r   __doc__r|   r   r   r/   r0   r1   rk   rk   p   s    Ka:!
r0   rk   c                     t        |  \  }}}}t        j                  |d      }t        j                  |d      }||t        |      t        |      fS Nr   dim)zipr,   stacklist)batchrm   rn   ro   target_wordss        r1   collate_seq_hint_batchr      sM    ?BE{<OZkk/q9OZQ/JJ](;T,=OOOr0   c                 z    t        |       dk  r| S |dk(  r| d d S |dk(  r| d t        dt        |              S | d d S )NrP   r   r      )rC   min)r[   modes     r1   make_hint_prefixr      sS    
4yA~CRy}'c!SY'((8Or0   c                 v    | d | D cg c]  }t        |       }}|sdg}t        j                  |      S c c}w )Nrd   )ri   r,   r   )prefixmax_lenchidss       r1   encode_hint_prefixr      s@    %+HW%5
6r;r?
6C
6dC   7s   6c                   2     e Zd ZdZd fd	Zd ZddZ xZS )HintSequencePredictorz;Two-stage predictor: first guess, then hint-assisted guess.c           	      z   t         |           t        j                  ||      | _        t        j                  d|      | _        t        j                  ||ddd      | _        t        j                  t        j                  ||      t        j                         t        j                  d      t        j                  ||            | _        t        j                  t        j                  ||z   |      t        j                         t        j                  d      t        j                  ||            | _        y )N   r   Tg?)
input_sizehidden_size
num_layersbatch_firstdropoutr   )superr|   nn	Embeddingword_embeddingshint_embeddingsGRUcontext_encoder
SequentialLinearReLUDropout
first_headsecond_head)rw   r$   r!   r"   r#   	__class__s        r1   r|   zHintSequencePredictor.__init__   s    !||KC!||B?!vv " 
 --IIj*-GGIJJsOIIj+.	
 ==IIj>1:>GGIJJsOIIj+.	
r0   c                     g }|D ]a  }| j                  |j                  | j                   j                  j                              }|j	                  |j                  d             c t        j                  |d      S r   )r   toweightr-   rv   meanr,   r   )rw   hint_prefix_batchhint_embeds
prefix_idsembs        r1   encode_hintz!HintSequencePredictor.encode_hint   sl    + 	0J&&z}}T5I5I5P5P5W5W'XYCsxxAx/	0 {{;A..r0   c                    | j                  |      }| j                  |      \  }}|d   }| j                  |      }||d |fS | j                  |      }t	        j
                  ||gd      }	| j                  |	      }
||
|fS )Nr   rP   r   )r   r   r   r   r,   catr   )rw   rm   r   context_embedr]   hiddenstatefirst_logits
hint_embedsecond_inputsecond_logitss              r1   forwardzHintSequencePredictor.forward   s    ,,_=((7	6r
u-$u,,%%&78
yy%!4!<((6]E11r0   )r
   r   r   r~   )r   r   r   r   r|   r   r   __classcell__)r   s   @r1   r   r      s    E
8/2r0   r   c                 ~    t         j                  d   d   }t         j                  D ]  \  }}}| |dz  k\  r|} |S  |S )Nr   rP         Y@cfgr+   )	first_accselected	thresholdprobr]   s        r1   hint_prob_for_accuracyr      sP    q!!$H!oo 	4	E))HO
 Or0   c                 ~    t         j                  d   d   }t         j                  D ]  \  }}}| |dz  k\  r|} |S  |S )Nr   r   r   r   )r   r   r   r]   r   s        r1   hint_mode_for_accuracyr      sP    q!!$H!oo 	1d	E))HO
 Or0   c                    t         j                  j                  | j                         |      }t	        j
                         }d}t        j                  d   d   }t        j                  d   d   }	d}
| j                  t        j                         t        |      D ]  }| j                          d}d}d}d}d}d}d}|D ]  \  }}}}|j                  t        j                        }|j                  d      j                  t        j                        }|j                           | |d       \  }}} |||      }t        |      }|j                         }||k(  j!                         }|}|j#                         j%                         }t        j&                  |z  dt        j&                  z
  |z  z   }||z
  }|j)                         |j+                  |      z  j#                          } |t        j,                  | z  z   }!|!j/                          t         j                  j0                  j3                  | j                         d	       |j5                          |j7                  d
      }"|"|k(  }#|#}$t        j8                  dt        j                        }%|"}&t;        j:                         |k  }'|'r|j                          |D (cg c]  }(t=        t?        |(|	             })}( | ||)      \  }}*} ||*|      }%|%j/                          t         j                  j0                  j3                  | j                         d	       |j5                          |*j7                  d
      }&|&|k(  }$|#|$z  }+||!j%                         |%j%                         z   z  }||j%                         |%j%                         z   z  }|t        j@                  |       r| j%                         n
t!        |       z  }||#jC                         j%                         z  }||$jC                         j%                         z  }||+jC                         j%                         z  }||jE                  d      z  }d|z  |z  },d|z  |z  }-d|z  |z  }. |tG        tI        |      d      z  }/|tG        tI        |      d      z  }0|tG        tI        |      d      z  }1| jK                          d}2d}3d}4d}5d}6t        jL                         5  |D ]  \  }}}}|j                  t        j                        }|j                  d      j                  t        j                        }|dk\  }'|'r0|D (cg c]  }(t=        t?        |(|	             })}( | ||)      \  }}*}n | |d       \  }}}d }* |||      }|*	 ||*|      n$t        j8                  dt        j                        }%|j7                  d
      }"|*|*j7                  d
      n|"}&|"|k(  }#|&|k(  }$|#|$z  }+|6|j%                         |%j%                         z   z  }6|2|#jC                         j%                         z  }2|3|$jC                         j%                         z  }3|4|+jC                         j%                         z  }4|5|jE                  d      z  }5 	 d d d        d|2z  |5z  }7d|3z  |5z  }8d|4z  |5z  }9|6tG        tI        |      d      z  }:|7}
tO        |
      }tQ        |
      }	tS        djU                  g d|dz    d| d|/dd|0dd|1dd,dd-dd.dd|dd|	 d|
dd|:dd|7dd|8dd |9dd!              | S c c}(w c c}(w # 1 sw Y   xY w)"N)r(   r   r   rP   r   r   )logitsg      ?)max_normr   )r-   r   r    zEpoch /z	 | Loss: .4fz | CE: z | RL: z | Train First: .2fz% | Train Hint: z% | Train Final: z% | HintP(next): z.3fz | HintMode(next): z | Val Gate First: z% | Val Loss: z | Val First: z% | Val Hint: z% | Val Final: %)+r,   optimAdam
parametersr   CrossEntropyLossr   r+   r   r-   ru   trainsqueeze	zero_gradr   r   floatr   itemr*   detachlog_probr)   backwardutilsclip_grad_norm_stepargmaxtensorrandomr   r   	is_tensorsumsizemaxrC   evalno_gradr   r   r9   join);modeltrain_loader
val_loaderr'   r(   	optimizer	criterionbaseline	hint_prob	hint_modeprev_val_first_accepoch
train_losstrain_cetrain_rlfirst_correctsecond_correctfinal_correcttotalcontext_idxrn   ro   r   r   r]   first_cedist1action1reward1shaped_rewardbatch_reward	advantagepolicy_loss
first_loss
first_predfirst_ok	second_ok	second_cesecond_preduse_hintr[   hint_prefixesr   final_okr   
second_acc	final_accavg_lossavg_ceavg_rl	val_first
val_second	val_final	val_totalval_lossval_first_accval_second_accval_final_accavg_val_losss;                                                              r1   train_modelr#     s>     !1!1!3 ;I##%IH"1%I"1%I	HHSZZv }

DP ;	6@K]L%..4K#++B/223::>J !!&{D!9L!Q z:H  |4EllnG*,335G#M(--/446L,,x73AVAV;VZf:ffH%0I%,,.1HHNNPPK!CMMK$??J!HHNN**5+;+;+=*LNN%,,,3J!Z/H I S<I$K}}2H##%co p[_!34DT94U!V p p&+K&G#=!%mZ@	""$..u/?/?/AC.P +22q29':5	)+H*//+inn.>>>J)..*:::Heook.J((*PUVaPbbHX\\^0022Mimmo2244NX\\^0022MZ__Q''E-5I/%7J-5Iw;	6x C$5q 99CL 1155CL 1155

	
		]]_ 	0HR 0DZ)nnSZZ8'//366szzB
$+gs$t_c%78Hy8Y%Z$tM$t5:;5V2L-).{D)A&L!Q$(M$\:>DQD]ImZ@chcocops|  }G  }G  dH	)00Q07
=J=Vm22q29\f%3':5	#i/X]]_y~~/??@X\\^0022	immo2244
X\\^0022	Z__Q//	10	06 	)I5+i7	)I5#c*oq"99**+=>	*+=>	Q Qf QU1WI QQ Qvh Q ' QcNQ")Q*0Q5<Q=CCLQIQ%c?Q*:Q;Ec:JQJ[Q\efi[jQkQ &c?Q +>Q ?H[Q I\Q ]oor[sQtQ &c*	Q +9	Q :Gs8K	QL	Q
 (,Q
 -<Q
 =J#;NQ
 OPQ	
m}
~ L_ !qR %u	0 	0s&   	].5A$]8]3
5D-]83]88^	c                 ,	   | j                          t        j                         }d}d}d}d}d}d}	t        d       t        d       t        d       t	        j
                         5  |D ]  \  }
}}}|
j                  t        j                        }
|j                  d      j                  t        j                        }|D cg c]  }t        t        |d             }} | |
|      \  }}} |||      } |||      }|j                  d	      }|j                  d	      }||k(  }||k(  }||z  }||j                         j                         z  }||j                         j                         z  }||j                         j                         z  }||j                  d      z  }||j                         |j                         z   z  }|	|j!                         j                         j                         d
|j!                         j                         j                         z  z   z  }	 	 d d d        d|z  |z  }d|z  |z  }d|z  |z  }|t#        t%        |      d      z  }|	|z  }t        d|d       t        d|dd| d| d       t        d|dd| d| d       t        d|dd| d| d       t        d|d       t        d       d}t	        j
                         5  |D ]f  \  }
}}}|D cg c]  }t        t        |d             }} | |
j                  t        j                        |      \  }}}|j                  d	      j'                         }|j                  d	      j'                         }t)        t+        dt%        |                  D ]  } ||    }!|j-                  ||    j                         d      }"|j-                  ||    j                         d      }#t        |!d      }$|"|!k(  rd}%n
|#|!k(  rd}%nd}%t        d|% ddj/                  ||    dd         d |" d!|$ d"|# d#|! d$       |dz  }|d%k\  s n |d%k\  sg n d d d        t        d       ||fS c c}w # 1 sw Y   IxY wc c}w # 1 sw Y   /xY w)&Nr   r   z=
============================================================u%   📊 TWO-STAGE NEXT-WORD TEST RESULTS<============================================================r   r   rP   r   r   r   zTest Loss: r   First Guess Accuracy: r   z% (r   rq   zHint-Assisted Accuracy: Final Accuracy After Hint: zAverage Reward: u   
🔍 Sample Predictions:   ?u   ✅ first guessu   🟡 hint recoveredu   ❌ missz  z:  z -> first='z	', hint='z', second='z	', true=''r   )r   r   r   r9   r,   r   r   r   r-   r   r   r   r   r   r   r   r   r   rC   r   ru   r   getr   )&r   test_loaderra   r   r   r   r  r  	test_lossreward_totalr  rn   ro   r   r[   r  r   r   r]   r  r  r  r  r  r  r  r   r  r  r  
avg_rewardshownr{   	true_word
first_wordsecond_wordhint_letterstatuss&                                         r1   
test_modelr8    s   	JJL##%IMNMEIL	/	
12	(O	 bDO 	b@K]L%..4K#++B/223::>JbnoZ^/0@|0TUoMo-2;-N*L- z:H!-<I%,,,3J'..1.5K!Z/H#z1I)+HX\\^0022Mimmo2244NX\\^0022MZ__Q''E(--/INN,<<=IHNN,002779D9??CTCXCXCZC_C_Ca<aaaL+	bb0 %-I'%/J%-I3s;/33H%J	K~
&'	"9S/]O1UG1
MN	$Z$4C7Gqq
QR	'	#c-%PQ
RS	Z,
-.	
&'E	 DO 	@K]LbnoZ^/0@|0TUoMo-2;>>#**3M}-]*L-%,,,3779J'..1.599;K3q#l"345 (O	%\\*Q-*<*<*>D
&ll;q>+>+>+@#F.y,G*.F I-2F'F388M!,<RS,A#B"C D(\;-{;-W`aj`kkln 
A:'( z7	< 
(OiA pb bP p sK   &AQ8Q3EQ8/R
?RD3R
R
R
3Q88RR

Rc                     t        d       t        d       t        d       t        t        j                        } t        d       t	        | d      \  }}}t        |      t        _        t        dt        j                   d       t        | |t        j                  d	
      }t        d       t        |      }t        d|z        }t        d|z        }||z
  |z
  }t        j                  j                  j                  ||||g      \  }	}
}t        |	t        j                  dt               }t        |
t        j                  dt               }t        |t        j                  dt               }t        dt        |	       dt        |
       dt        |              t        d       t#        t        j                  t        j$                  t        j&                  t        j(                        }t        |       t        d       t+        |||t        j,                  t        j.                        }t        d       t1        |||      \  }}t        j2                  |j5                         ||t        j$                  t        j&                  t        j(                  t        j                  t        j                  ddd       t        d       t        d|dd       t        d |dd       y )!Nu%   🧠 Two-Stage Next-Word Reward Modelr%  u'   
📂 Step 1: Loading markdown files...u$   
📚 Step 2: Building vocabulary...i  )rY   u9   
🖼️ Step 3: Creating sequence-hint dataset (seq_len=z)...i@  )r%   rx   u   
📊 Step 4: Splitting data...r   r   T)r&   shuffle
collate_fnFz	  Train: z | Val: z	 | Test: u"   
🏗️ Step 5: Building model...)r$   r!   r"   r#   u0   
🚀 Step 6: Training two-stage reward model...)r'   r(   u   
🧪 Step 7: Testing...)r!   r"   r#   r$   r%   )model_state_dictr`   ra   configztwo_stage_reward_sequence.ptu3   
💾 Model saved to 'two_stage_reward_sequence.pt'r&  r   r   r'  )r9   rL   r   r    rb   rC   r$   r%   rk   rU   r,   r   datarandom_splitr   r&   r   r   r!   r"   r#   r#  r'   r(   r8  save
state_dict)rE   r`   ra   r^   datasetr  
train_sizeval_size	test_sizetrain_datasetval_datasettest_datasetr   r   r.  r   r   r  s                     r1   mainrI    sU   	
12	(O	
450E	
12&6uT&R#Hh+&CO	Fs{{mSW
XY!%3;;TYZG	
,-LES5[!J4%< H
"X-I/4{{/?/?/L/L*h	20,M; mPTawxLKCNNE^tuJ\cnne`vwK	Ic-()#k2B1C9SQ]M^L_
`a	
/0!OO-->>))	E 
%L	
=>|Z

svvVE	
%&%e[(CIy	JJ % 0 0 2   ]]!nn"%"4"4";;		
 	'  

@A	"9S/
34	'	#a
89r0   __main__)i  )r   )r   r   )&r   r?   r   collectionsr   pathlibr   numpynpr,   torch.nnr   torch.nn.functional
functionalFtorch.distributionsr   torch.utils.datar   r   r	   r   rL   r:   rb   ri   rk   r   r   r   Moduler   r   r   r#  r8  rI  r   r/   r0   r1   <module>rV     s    
  #       + 0J J: h.<+**
' *
ZP!32BII 32lIXT n?:D zF r0   