From 8c0dd30f7a97e0a1582510e5d048ed448be61bfc Mon Sep 17 00:00:00 2001 From: Genstark Date: Thu, 2 Oct 2025 16:06:57 +0530 Subject: [PATCH 1/5] feat(training-data): expand pre-training and chat training datasets with additional examples --- src/main.rs | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/main.rs b/src/main.rs index b08e920..5652414 100644 --- a/src/main.rs +++ b/src/main.rs @@ -58,6 +58,26 @@ fn main() { "Seasons change throughout the year ", "Animals eat food to survive ", "Plants need sunlight and water to grow ", + "Deserts are dry regions with little rainfall ", + "The Earth revolves around the sun once a year ", + "Tornadoes are fast-spinning columns of air ", + "Rainbows form when light refracts through water droplets ", + "Caves are natural underground spaces ", + "Bees collect nectar to make honey ", + "Lava flows from erupting volcanoes ", + "Thunder follows lightning during a storm ", + "Leaves change color in autumn ", + "The ocean is salty and covers most of Earth ", + "Spiders spin webs to catch insects ", + "Dolphins are intelligent marine mammals ", + "Butterflies emerge from cocoons ", + "Seeds grow into plants when given water and sunlight ", + "The sky appears blue due to scattered sunlight ", + "Airplanes fly by generating lift with their wings ", + "Tides are caused by the moon's gravity ", + "Bamboo grows quickly and is used in construction ", + "Crystals form through a natural solidification process ", + "Caterpillars turn into butterflies through metamorphosis " ]; let chat_training_data = vec![ @@ -106,6 +126,23 @@ fn main() { ("User: How do compasses work? Assistant: Compasses work by aligning a magnetic needle with the earth's magnetic field "), ("User: What do vaccines do? Assistant: Vaccines help the immune system recognize and fight off specific pathogens "), + ("User: What causes earthquakes? Assistant: Earthquakes are caused by sudden movements of tectonic plates beneath Earth's surface "), + ("User: How are clouds formed? Assistant: Clouds form when moist air rises and cools, causing water vapor to condense "), + ("User: Why do leaves fall in autumn? Assistant: Leaves fall because trees prepare for winter by conserving energy and water "), + ("User: How do magnets work? Assistant: Magnets work by generating a magnetic field that attracts certain materials like iron "), + ("User: What are fossils? Assistant: Fossils are preserved remains or impressions of ancient organisms "), + ("User: What is evaporation? Assistant: Evaporation is the process where liquid turns into vapor due to heat "), + ("User: What is the ozone layer? Assistant: The ozone layer is a region of Earth's atmosphere that absorbs most of the sun's ultraviolet radiation "), + ("User: Why do animals hibernate? Assistant: Animals hibernate to conserve energy during times of cold and food scarcity "), + ("User: What is a black hole? Assistant: A black hole is a region in space where gravity is so strong that nothing can escape from it "), + ("User: What is the water cycle? Assistant: The water cycle describes how water moves through evaporation, condensation, precipitation, and collection "), + ("User: What is a constellation? Assistant: A constellation is a group of stars that form a recognizable pattern in the night sky "), + ("User: What is renewable energy? Assistant: Renewable energy comes from sources that naturally replenish, such as wind, solar, and water "), + ("User: How do ants live? Assistant: Ants live in colonies with structured roles, including workers, soldiers, and a queen "), + ("User: What is a galaxy? Assistant: A galaxy is a massive system of stars, gas, dust, and dark matter held together by gravity "), + ("User: Why do we see different phases of the moon? Assistant: The moon's phases result from its position relative to Earth and the sun ") + + // Greeting and conversational patterns ("User: Hello! Assistant: Hello! How can I help you today? "), ("User: Hi! Assistant: Hi there! What can I do for you? "), @@ -120,6 +157,13 @@ fn main() { ("User: Can you explain that? Assistant: Of course! I'd be happy to explain in more detail "), ("User: I don't understand. Assistant: No problem! Let me try to explain it differently "), ("User: What do you think? Assistant: Based on the information available, I think it's important to consider multiple perspectives "), + ("User: Thanks for your help. Assistant: You're very welcome! Let me know if you have more questions "), + ("User: That makes sense. Assistant: I'm glad to hear that! Is there anything else you'd like to explore? "), + ("User: Interesting! Assistant: I agree—it's fascinating! Would you like to dive deeper into the topic? "), + ("User: Could you give me an example? Assistant: Sure! Here's an example that might help clarify things "), + ("User: That's confusing. Assistant: No worries! Let me try to explain it another way "), + ("User: Do you know more about that? Assistant: Absolutely! Here's some more information that might help ") + ]; // Process all training examples for vocabulary From cfe95533a372e8744be3444889feb117c20c5f9d Mon Sep 17 00:00:00 2001 From: Genstark Date: Thu, 2 Oct 2025 22:53:04 +0530 Subject: [PATCH 2/5] feat(training-data): add new examples to pre-training and chat training datasets --- data/chat_training_data.json | 23 +++++- data/pretraining_data.json | 22 +++++- src/main.rs | 138 +---------------------------------- 3 files changed, 44 insertions(+), 139 deletions(-) diff --git a/data/chat_training_data.json b/data/chat_training_data.json index 4867459..17b9f83 100644 --- a/data/chat_training_data.json +++ b/data/chat_training_data.json @@ -51,5 +51,26 @@ "User: Thank you. Assistant: You're welcome! I'm happy to help ", "User: Can you explain that? Assistant: Of course! I'd be happy to explain in more detail ", "User: I don't understand. Assistant: No problem! Let me try to explain it differently ", - "User: What do you think? Assistant: Based on the information available, I think it's important to consider multiple perspectives " + "User: What do you think? Assistant: Based on the information available, I think it's important to consider multiple perspectives ", + "User: What causes earthquakes? Assistant: Earthquakes are caused by sudden movements of tectonic plates beneath Earth's surface ", + "User: How are clouds formed? Assistant: Clouds form when moist air rises and cools, causing water vapor to condense ", + "User: Why do leaves fall in autumn? Assistant: Leaves fall because trees prepare for winter by conserving energy and water ", + "User: How do magnets work? Assistant: Magnets work by generating a magnetic field that attracts certain materials like iron ", + "User: What are fossils? Assistant: Fossils are preserved remains or impressions of ancient organisms ", + "User: What is evaporation? Assistant: Evaporation is the process where liquid turns into vapor due to heat ", + "User: What is the ozone layer? Assistant: The ozone layer is a region of Earth's atmosphere that absorbs most of the sun's ultraviolet radiation ", + "User: Why do animals hibernate? Assistant: Animals hibernate to conserve energy during times of cold and food scarcity ", + "User: What is a black hole? Assistant: A black hole is a region in space where gravity is so strong that nothing can escape from it ", + "User: What is the water cycle? Assistant: The water cycle describes how water moves through evaporation, condensation, precipitation, and collection ", + "User: What is a constellation? Assistant: A constellation is a group of stars that form a recognizable pattern in the night sky ", + "User: What is renewable energy? Assistant: Renewable energy comes from sources that naturally replenish, such as wind, solar, and water ", + "User: How do ants live? Assistant: Ants live in colonies with structured roles, including workers, soldiers, and a queen ", + "User: What is a galaxy? Assistant: A galaxy is a massive system of stars, gas, dust, and dark matter held together by gravity ", + "User: Why do we see different phases of the moon? Assistant: The moon's phases result from its position relative to Earth and the sun ", + "User: Thanks for your help. Assistant: You're very welcome! Let me know if you have more questions ", + "User: That makes sense. Assistant: I'm glad to hear that! Is there anything else you'd like to explore? ", + "User: Interesting! Assistant: I agree—it's fascinating! Would you like to dive deeper into the topic? ", + "User: Could you give me an example? Assistant: Sure! Here's an example that might help clarify things ", + "User: That's confusing. Assistant: No worries! Let me try to explain it another way ", + "User: Do you know more about that? Assistant: Absolutely! Here's some more information that might help " ] diff --git a/data/pretraining_data.json b/data/pretraining_data.json index d5c8c93..c7e2aac 100644 --- a/data/pretraining_data.json +++ b/data/pretraining_data.json @@ -23,5 +23,25 @@ "Storms bring rain and strong winds ", "Seasons change throughout the year ", "Animals eat food to survive ", - "Plants need sunlight and water to grow " + "Plants need sunlight and water to grow ", + "Deserts are dry regions with little rainfall ", + "The Earth revolves around the sun once a year ", + "Tornadoes are fast-spinning columns of air ", + "Rainbows form when light refracts through water droplets ", + "Caves are natural underground spaces ", + "Bees collect nectar to make honey ", + "Lava flows from erupting volcanoes ", + "Thunder follows lightning during a storm ", + "Leaves change color in autumn ", + "The ocean is salty and covers most of Earth ", + "Spiders spin webs to catch insects ", + "Dolphins are intelligent marine mammals ", + "Butterflies emerge from cocoons ", + "Seeds grow into plants when given water and sunlight ", + "The sky appears blue due to scattered sunlight ", + "Airplanes fly by generating lift with their wings ", + "Tides are caused by the moon's gravity ", + "Bamboo grows quickly and is used in construction ", + "Crystals form through a natural solidification process ", + "Caterpillars turn into butterflies through metamorphosis " ] diff --git a/src/main.rs b/src/main.rs index ea54a0d..cc36658 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,142 +35,6 @@ fn main() { DatasetType::JSON, ); // Placeholder, not used in this example - - // Pre-training data - simple text completion patterns - let pretraining_data = vec![ - "The sun rises in the east and sets in the west ", - "Water flows downhill due to gravity ", - "Birds fly through the air using their wings ", - "Fish swim in rivers, lakes, and oceans ", - "Trees grow tall and produce leaves ", - "Rain falls from clouds in the sky ", - "Fire is hot and produces light ", - "Ice is frozen water that melts when heated ", - "Mountains are tall and rocky formations ", - "The moon orbits around planet Earth ", - "Flowers bloom in spring and summer ", - "Snow is cold and white ", - "Wind moves air from place to place ", - "Rivers flow into larger bodies of water ", - "Sand is found on beaches and in deserts ", - "Grass grows in fields and yards ", - "Rocks are hard and can be different colors ", - "Stars shine bright in the night sky ", - "Waves move across the surface of water ", - "Clouds form when water vapor rises ", - "Lightning is bright and makes thunder ", - "Storms bring rain and strong winds ", - "Seasons change throughout the year ", - "Animals eat food to survive ", - "Plants need sunlight and water to grow ", - "Deserts are dry regions with little rainfall ", - "The Earth revolves around the sun once a year ", - "Tornadoes are fast-spinning columns of air ", - "Rainbows form when light refracts through water droplets ", - "Caves are natural underground spaces ", - "Bees collect nectar to make honey ", - "Lava flows from erupting volcanoes ", - "Thunder follows lightning during a storm ", - "Leaves change color in autumn ", - "The ocean is salty and covers most of Earth ", - "Spiders spin webs to catch insects ", - "Dolphins are intelligent marine mammals ", - "Butterflies emerge from cocoons ", - "Seeds grow into plants when given water and sunlight ", - "The sky appears blue due to scattered sunlight ", - "Airplanes fly by generating lift with their wings ", - "Tides are caused by the moon's gravity ", - "Bamboo grows quickly and is used in construction ", - "Crystals form through a natural solidification process ", - "Caterpillars turn into butterflies through metamorphosis " - ]; - - let chat_training_data = vec![ - // Conversational instruction-following data - ("User: What causes rain? Assistant: Rain is caused by water vapor in clouds condensing into droplets that become too heavy to remain airborne "), - ("User: How do mountains form? Assistant: Mountains are formed through tectonic forces or volcanism over long geological time periods "), - ("User: What is the Amazon rainforest? Assistant: The Amazon rainforest is one of the most biodiverse places on earth, home to countless species "), - ("User: At what temperature does water boil? Assistant: Water boils at 100 degrees celsius at standard atmospheric pressure "), - ("User: How long does it take the moon to orbit Earth? Assistant: The moon orbits the earth approximately every 27.3 days "), - ("User: What is photosynthesis? Assistant: Photosynthesis is the process by which green plants use sunlight to synthesize food from carbon dioxide "), - ("User: How does gravity work? Assistant: Gravity is a force that attracts two bodies toward each other based on their mass "), - ("User: How many neurons are in the human brain? Assistant: The human brain contains about 86 billion neurons that transmit information throughout the body "), - ("User: What is electricity? Assistant: Electricity is the flow of electrons through a conductor, often used to power devices "), - ("User: What is climate change? Assistant: Climate change refers to long-term shifts in temperatures and weather patterns on Earth "), - - ("User: How long do oak trees live? Assistant: Oak trees can live for hundreds of years and produce acorns as their fruit "), - ("User: What happened to Pluto? Assistant: Pluto was reclassified from a planet to a dwarf planet in 2006 by astronomers "), - ("User: How is glass made? Assistant: Glass is made by heating sand, soda ash, and limestone to very high temperatures until they melt "), - ("User: What do volcanoes do? Assistant: Volcanoes can erupt with lava, ash, and gases, altering landscapes and ecosystems "), - ("User: Why was the Great Wall built? Assistant: The great wall of china was built to protect ancient china from invasions "), - ("User: What are penguins like? Assistant: Penguins are flightless birds that are well adapted to life in cold environments "), - ("User: How much rain do deserts get? Assistant: Deserts receive less than 250 millimeters of precipitation each year "), - ("User: What is Jupiter like? Assistant: Jupiter is the largest planet in our solar system and has dozens of moons "), - ("User: How fast does light travel? Assistant: Light travels at approximately 299,792 kilometers per second in a vacuum "), - ("User: What is gold used for? Assistant: Gold is a dense, soft metal often used in jewelry and electronics due to its conductivity "), - - ("User: What covers most of Earth? Assistant: Most of the earth's surface is covered by water, primarily in oceans "), - ("User: How do bicycles work? Assistant: Bicycles are an efficient mode of transport that convert human energy into motion through gears "), - ("User: How is chocolate made? Assistant: Chocolate is made from roasted and ground cacao seeds, often sweetened and flavored "), - ("User: What is the internet? Assistant: The internet is a global network that allows for digital communication and information sharing "), - ("User: How do wind turbines work? Assistant: Wind energy is harnessed using turbines that convert wind motion into electricity "), - ("User: What are cats like? Assistant: Cats are domesticated mammals known for their independence and hunting instincts "), - ("User: How do languages change? Assistant: Languages evolve over time through cultural, social, and technological influences "), - ("User: What did the printing press do? Assistant: The printing press revolutionized the spread of information in the 15th century "), - ("User: What is sound? Assistant: Sound is a vibration that travels through air, water, or solid materials to reach our ears "), - ("User: Why is carbon important? Assistant: Carbon is an essential element in organic chemistry, forming the basis of all life on Earth "), - - ("User: What was the Library of Alexandria? Assistant: The library of alexandria was one of the most significant libraries of the ancient world "), - ("User: What do honeybees do? Assistant: Honeybees play a vital role in pollination, which supports ecosystems and agriculture "), - ("User: Are electric cars better? Assistant: Electric vehicles produce less air pollution than traditional gasoline-powered cars "), - ("User: How do you make bread? Assistant: Bread is typically made from flour, water, yeast, and salt through a baking process "), - ("User: What is the Sahara desert? Assistant: The sahara desert is the largest hot desert in the world, spanning multiple countries "), - ("User: What are renewable resources? Assistant: Renewable resources replenish naturally and include sunlight, wind, and water "), - ("User: What are eclipses? Assistant: Eclipses occur when one celestial body moves into the shadow of another "), - ("User: How are language models trained? Assistant: Language models are trained using vast amounts of text to learn patterns in language "), - ("User: How do compasses work? Assistant: Compasses work by aligning a magnetic needle with the earth's magnetic field "), - ("User: What do vaccines do? Assistant: Vaccines help the immune system recognize and fight off specific pathogens "), - - ("User: What causes earthquakes? Assistant: Earthquakes are caused by sudden movements of tectonic plates beneath Earth's surface "), - ("User: How are clouds formed? Assistant: Clouds form when moist air rises and cools, causing water vapor to condense "), - ("User: Why do leaves fall in autumn? Assistant: Leaves fall because trees prepare for winter by conserving energy and water "), - ("User: How do magnets work? Assistant: Magnets work by generating a magnetic field that attracts certain materials like iron "), - ("User: What are fossils? Assistant: Fossils are preserved remains or impressions of ancient organisms "), - ("User: What is evaporation? Assistant: Evaporation is the process where liquid turns into vapor due to heat "), - ("User: What is the ozone layer? Assistant: The ozone layer is a region of Earth's atmosphere that absorbs most of the sun's ultraviolet radiation "), - ("User: Why do animals hibernate? Assistant: Animals hibernate to conserve energy during times of cold and food scarcity "), - ("User: What is a black hole? Assistant: A black hole is a region in space where gravity is so strong that nothing can escape from it "), - ("User: What is the water cycle? Assistant: The water cycle describes how water moves through evaporation, condensation, precipitation, and collection "), - ("User: What is a constellation? Assistant: A constellation is a group of stars that form a recognizable pattern in the night sky "), - ("User: What is renewable energy? Assistant: Renewable energy comes from sources that naturally replenish, such as wind, solar, and water "), - ("User: How do ants live? Assistant: Ants live in colonies with structured roles, including workers, soldiers, and a queen "), - ("User: What is a galaxy? Assistant: A galaxy is a massive system of stars, gas, dust, and dark matter held together by gravity "), - ("User: Why do we see different phases of the moon? Assistant: The moon's phases result from its position relative to Earth and the sun ") - - - // Greeting and conversational patterns - ("User: Hello! Assistant: Hello! How can I help you today? "), - ("User: Hi! Assistant: Hi there! What can I do for you? "), - ("User: Hey! Assistant: Hey! How's it going? What would you like to know? "), - ("User: Good morning! Assistant: Good morning! Hope you're having a great day. How can I assist you? "), - ("User: Good afternoon! Assistant: Good afternoon! What can I help you with today? "), - ("User: Good evening! Assistant: Good evening! How may I help you tonight? "), - ("User: How are you? Assistant: I'm doing well, thank you for asking! How can I help you today? "), - ("User: What's up? Assistant: Not much, just here to help! What would you like to know? "), - ("User: Greetings! Assistant: Greetings! It's nice to meet you. How can I be of assistance? "), - ("User: Thank you. Assistant: You're welcome! I'm happy to help "), - ("User: Can you explain that? Assistant: Of course! I'd be happy to explain in more detail "), - ("User: I don't understand. Assistant: No problem! Let me try to explain it differently "), - ("User: What do you think? Assistant: Based on the information available, I think it's important to consider multiple perspectives "), - ("User: Thanks for your help. Assistant: You're very welcome! Let me know if you have more questions "), - ("User: That makes sense. Assistant: I'm glad to hear that! Is there anything else you'd like to explore? "), - ("User: Interesting! Assistant: I agree—it's fascinating! Would you like to dive deeper into the topic? "), - ("User: Could you give me an example? Assistant: Sure! Here's an example that might help clarify things "), - ("User: That's confusing. Assistant: No worries! Let me try to explain it another way "), - ("User: Do you know more about that? Assistant: Absolutely! Here's some more information that might help ") - - ]; - // Process all training examples for vocabulary // First process pre-training data for text in &dataset.pretraining_data { @@ -320,4 +184,4 @@ fn main() { let prediction = llm.predict(&formatted_input); println!("Model output: {}", prediction); } -} +} \ No newline at end of file From abe4248e5efe0f0bfca4ec79dd1a61ee3ca35de0 Mon Sep 17 00:00:00 2001 From: Genstark Date: Fri, 3 Oct 2025 15:25:04 +0530 Subject: [PATCH 3/5] refactor --- src/main.rs | 79 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 28 deletions(-) diff --git a/src/main.rs b/src/main.rs index cc36658..a23c563 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ use crate::vocab::Vocab; use ::llm::{EMBEDDING_DIM, HIDDEN_DIM, MAX_SEQ_LEN}; use dataset_loader::{Dataset, DatasetType}; +// Import local modules that make up different parts of the model mod adam; mod dataset_loader; mod embeddings; @@ -20,26 +21,26 @@ mod transformer; mod vocab; fn main() { - // Mock input - test conversational format + // Mock input used to test the model before and after training let string = String::from("User: How do mountains form?"); - // Extract all unique words from training data to create vocabulary + // Create a hash set to collect all unique tokens (words and punctuation) let mut vocab_set = std::collections::HashSet::new(); - // Add end of sequence token + // Add special end-of-sequence token to vocabulary vocab_set.insert("".to_string()); + // === Load dataset === let dataset = Dataset::new( String::from("data/pretraining_data.json"), String::from("data/chat_training_data.json"), DatasetType::JSON, - ); // Placeholder, not used in this example + ); // This loads pretraining and chat fine-tuning data - // Process all training examples for vocabulary - // First process pre-training data + // === Build vocabulary from pre-training data === for text in &dataset.pretraining_data { for word in text.split_whitespace() { - // Handle punctuation by splitting it from words + // Split punctuation from words (e.g., "hello," → "hello" and ",") let mut current = String::new(); for c in word.chars() { if c.is_ascii_punctuation() { @@ -58,17 +59,15 @@ fn main() { } } - // Then process chat training data + // === Build vocabulary from chat (instruction-tuning) data === for row in &dataset.chat_training_data { - // Add words from outputs for word in row.split_whitespace() { - // Handle punctuation by splitting it from words let mut current = String::new(); for c in word.chars() { if c.is_ascii_punctuation() { if !current.is_empty() { - vocab_set.insert(current.clone()); // Clone to avoid moving - current.clear(); // Use clear() instead of String::new() + vocab_set.insert(current.clone()); + current.clear(); } vocab_set.insert(c.to_string()); } else { @@ -81,16 +80,29 @@ fn main() { } } + // Convert vocabulary set into a sorted vector for deterministic order let mut vocab_words: Vec = vocab_set.into_iter().collect(); - vocab_words.sort(); // Sort for deterministic ordering + vocab_words.sort(); + + // Convert Vec to Vec<&str> because Vocab expects string slices let vocab_words_refs: Vec<&str> = vocab_words.iter().map(|s: &String| s.as_str()).collect(); + + // Build the vocabulary structure let vocab = Vocab::new(vocab_words_refs); + // === Build Transformer-based model === + // These represent multiple stacked Transformer layers let transformer_block_1 = TransformerBlock::new(EMBEDDING_DIM, HIDDEN_DIM); let transformer_block_2 = TransformerBlock::new(EMBEDDING_DIM, HIDDEN_DIM); let transformer_block_3 = TransformerBlock::new(EMBEDDING_DIM, HIDDEN_DIM); + + // Projection layer: maps hidden state to vocabulary logits let output_projection = OutputProjection::new(EMBEDDING_DIM, vocab.words.len()); + + // Embedding layer: turns tokens into dense vectors let embeddings = Embeddings::new(vocab.clone()); + + // Create the full LLM by stacking components in order let mut llm = LLM::new( vocab, vec![ @@ -102,19 +114,21 @@ fn main() { ], ); + // === Print model information === println!("\n=== MODEL INFORMATION ==="); println!("Network architecture: {}", llm.network_description()); println!( "Model configuration -> max_seq_len: {}, embedding_dim: {}, hidden_dim: {}", MAX_SEQ_LEN, EMBEDDING_DIM, HIDDEN_DIM ); - println!("Total parameters: {}", llm.total_parameters()); + // === Test before any training === println!("\n=== BEFORE TRAINING ==="); println!("Input: {}", string); println!("Output: {}", llm.predict(&string)); + // === Pre-training phase === println!("\n=== PRE-TRAINING MODEL ==="); println!( "Pre-training on {} examples for {} epochs with learning rate {}", @@ -123,20 +137,17 @@ fn main() { 0.0005 ); + // Collect pre-training examples as slices let pretraining_examples: Vec<&str> = dataset .pretraining_data .iter() .map(|s| s.as_str()) .collect(); - let chat_training_examples: Vec<&str> = dataset - .chat_training_data - .iter() - .map(|s| s.as_str()) - .collect(); - + // Train the model on pretraining data llm.train(pretraining_examples, 100, 0.0005); + // === Instruction tuning (fine-tuning for chat) === println!("\n=== INSTRUCTION TUNING ==="); println!( "Instruction tuning on {} examples for {} epochs with learning rate {}", @@ -145,43 +156,55 @@ fn main() { 0.0001 ); - llm.train(chat_training_examples, 100, 0.0001); // Much lower learning rate for stability + // Collect chat training examples as slices + let chat_training_examples: Vec<&str> = dataset + .chat_training_data + .iter() + .map(|s| s.as_str()) + .collect(); + + // Train again on chat data with a smaller learning rate for stability + llm.train(chat_training_examples, 100, 0.0001); + // === Test after training === println!("\n=== AFTER TRAINING ==="); println!("Input: {}", string); let result = llm.predict(&string); println!("Output: {}", result); println!("======================\n"); - // Interactive mode for user input + // === Interactive loop === println!("\n--- Interactive Mode ---"); println!("Type a prompt and press Enter to generate text."); println!("Type 'exit' to quit."); let mut input = String::new(); loop { - // Clear the input string + // Clear input buffer input.clear(); - // Prompt for user input + // Print prompt without newline print!("\nEnter prompt: "); std::io::stdout().flush().unwrap(); - // Read user input + // Read line from stdin std::io::stdin() .read_line(&mut input) .expect("Failed to read input"); - // Trim whitespace and check for exit command + // Remove surrounding whitespace let trimmed_input = input.trim(); + + // Check for exit command if trimmed_input.eq_ignore_ascii_case("exit") { println!("Exiting interactive mode."); break; } - // Generate prediction based on user input with "User:" prefix + // Add "User:" prefix so model understands the format + // Generate response from model let formatted_input = format!("User: {}", trimmed_input); let prediction = llm.predict(&formatted_input); println!("Model output: {}", prediction); } -} \ No newline at end of file +} From 384ed517568cf007d3c1766c151bef4b2f391da0 Mon Sep 17 00:00:00 2001 From: Genstark Date: Wed, 8 Oct 2025 21:24:36 +0530 Subject: [PATCH 4/5] fix: correct imports for LLM, Embeddings, Vocab, TransformerBlock, and OutputProjection --- src/main.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index a23c563..2447be0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,14 +1,11 @@ use std::io::Write; use crate::embeddings::Embeddings; -use crate::llm::LLM; +use crate::llm::{LLM, EMBEDDING_DIM, HIDDEN_DIM, MAX_SEQ_LEN}; use crate::output_projection::OutputProjection; use crate::transformer::TransformerBlock; use crate::vocab::Vocab; -use ::llm::{EMBEDDING_DIM, HIDDEN_DIM, MAX_SEQ_LEN}; -use dataset_loader::{Dataset, DatasetType}; -// Import local modules that make up different parts of the model mod adam; mod dataset_loader; mod embeddings; From 7ae71616e5437380b441d5a327c7754171bde2fc Mon Sep 17 00:00:00 2001 From: Genstark Date: Thu, 27 Nov 2025 17:26:17 +0530 Subject: [PATCH 5/5] refactor: consolidate imports and remove unused module declarations --- src/main.rs | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2447be0..ffc5cec 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,21 +1,13 @@ use std::io::Write; -use crate::embeddings::Embeddings; -use crate::llm::{LLM, EMBEDDING_DIM, HIDDEN_DIM, MAX_SEQ_LEN}; -use crate::output_projection::OutputProjection; -use crate::transformer::TransformerBlock; -use crate::vocab::Vocab; - -mod adam; -mod dataset_loader; -mod embeddings; -mod feed_forward; -mod layer_norm; -mod llm; -mod output_projection; -mod self_attention; -mod transformer; -mod vocab; +use llm::{ + EMBEDDING_DIM, HIDDEN_DIM, MAX_SEQ_LEN, Dataset, DatasetType, + Embeddings, LLM, OutputProjection, TransformerBlock, Vocab, +}; + +// We use the library crate (`llm`) defined in `src/lib.rs` instead of re-declaring +// modules inside the binary. The library re-exports Dataset/DatasetType and the +// configuration constants (MAX_SEQ_LEN, EMBEDDING_DIM, HIDDEN_DIM). fn main() { // Mock input used to test the model before and after training