diff --git a/ecomm-demo/.env.example b/ecomm-demo/.env.example new file mode 100644 index 0000000..eb9d199 --- /dev/null +++ b/ecomm-demo/.env.example @@ -0,0 +1,3 @@ +# Apify API Token +# Get your token at https://console.apify.com/account#/integrations +APIFY_TOKEN=your_apify_token_here diff --git a/ecomm-demo/.gitignore b/ecomm-demo/.gitignore index 2017053..6fe06f8 100644 --- a/ecomm-demo/.gitignore +++ b/ecomm-demo/.gitignore @@ -31,7 +31,11 @@ yarn-error.log* .pnpm-debug.log* # env files (can opt-in for committing if needed) -.env* +.env +.env.local +.env.development.local +.env.test.local +.env.production.local # vercel .vercel @@ -42,5 +46,4 @@ next-env.d.ts # internal task-manager.md -PRD.md -README.md \ No newline at end of file +PRD.md \ No newline at end of file diff --git a/ecomm-demo/README.md b/ecomm-demo/README.md new file mode 100644 index 0000000..f9760a6 --- /dev/null +++ b/ecomm-demo/README.md @@ -0,0 +1,156 @@ +# E-Commerce Demo with Apify Integration + +This Next.js application demonstrates how to integrate the Apify E-Commerce Scraping Tool to scrape real-time product data from e-commerce websites. + +## Features + +- 🔍 **Real-time Product Search**: Search for products using keywords +- 🚀 **Apify Integration**: Powered by Apify's E-Commerce Scraping Tool +- 📊 **Multiple Views**: View products as statistics, table, or cards +- 💰 **Dynamic Pricing**: Automatically calculates average prices +- 🎨 **Modern UI**: Built with Next.js, Tailwind CSS, and shadcn/ui + +## Prerequisites + +- Node.js 20 or higher +- An Apify account (free tier available) +- Apify API token + +## Setup Instructions + +### 1. Get Your Apify API Token + +1. Go to [Apify Console](https://console.apify.com/account#/integrations) +2. Create a free account if you don't have one +3. Navigate to Settings → Integrations +4. Copy your API token + +### 2. Configure Environment Variables + +1. Copy the example environment file: + ```bash + cp .env.example .env.local + ``` + +2. Edit `.env.local` and add your Apify token: + ```env + APIFY_TOKEN=your_actual_token_here + ``` + +### 3. Install Dependencies + +```bash +npm install +``` + +### 4. Run the Application + +For development: +```bash +npm run dev +``` + +For production: +```bash +npm run build +npm start +``` + +Open [http://localhost:3000](http://localhost:3000) in your browser. + +## How to Use + +1. **Enter a Search Query**: Type a product keyword (e.g., "laptop", "headphones") into the search bar +2. **Click Submit**: The app will call the Apify E-Commerce Scraping Tool to scrape products from Amazon +3. **View Results**: Products will be displayed in three sections: + - **Statistics Cards**: Total products, average price, and data source + - **Product Table**: Detailed tabular view with images, titles, prices, descriptions, and URLs + - **Product Cards**: Visual card layout for browsing products + +## Architecture + +### Components + +- **`app/page.tsx`**: Main page with search functionality and state management +- **`app/api/scrape/route.ts`**: API route that calls the Apify Actor (server-side only) +- **`components/SearchBar.tsx`**: Search input component +- **`components/StatsCards.tsx`**: Statistics display component +- **`components/ProductTable.tsx`**: Tabular product view +- **`components/ProductCards.tsx`**: Card-based product view + +### Data Flow + +1. User enters search query → `SearchBar` component +2. Client sends POST request → `/api/scrape` API route +3. API route calls Apify Actor → `apify/e-commerce-scraping-tool` +4. Actor scrapes products from Amazon → Returns dataset +5. API route transforms data → Returns to client +6. Client updates state → Components re-render with new data + +### Security + +- **APIFY_TOKEN** is stored server-side only in environment variables +- API route handles all Apify Actor calls, keeping credentials secure +- Client never has direct access to the Apify token + +## Configuration + +### Scraping Settings + +In `app/api/scrape/route.ts`, you can customize: + +- **Marketplaces**: Currently set to Amazon US, but supports many others +- **Max Results**: Limited to 20 products for demo (adjustable) +- **Scrape Mode**: Set to "AUTO" (can be "BROWSER" or "HTTP") + +### Supported Marketplaces + +The E-Commerce Scraping Tool supports many marketplaces including: +- Amazon (all regions) +- Walmart +- eBay +- Alibaba +- IKEA +- And many more + +To change the marketplace, edit the `marketplaces` array in the API route. + +## Cost Information + +The E-Commerce Scraping Tool uses Apify's pay-per-event pricing model: +- **Actor Start**: ~$0.0001 +- **Product Listing**: ~$0.0005 per page +- **Product Details**: ~$0.006 per product + +Apify offers a free tier with monthly credits. See [Apify Pricing](https://apify.com/pricing) for details. + +## Troubleshooting + +### "APIFY_TOKEN not configured" Error + +Make sure you have: +1. Created a `.env.local` file +2. Added your `APIFY_TOKEN` +3. Restarted the development server + +### No Products Found + +Try: +- Using different search keywords +- Checking your Apify account quota +- Viewing the Apify Console run details (link provided after search) + +### Image Not Loading + +The app is configured to allow images from common e-commerce domains. If you see broken images, check the `next.config.ts` file and add the required domain. + +## Learn More + +- [Apify Documentation](https://docs.apify.com) +- [E-Commerce Scraping Tool](https://apify.com/apify/e-commerce-scraping-tool) +- [Next.js Documentation](https://nextjs.org/docs) +- [Apify JavaScript SDK](https://docs.apify.com/sdk/js/) + +## License + +MIT diff --git a/ecomm-demo/app/api/scrape/route.ts b/ecomm-demo/app/api/scrape/route.ts new file mode 100644 index 0000000..7ca1620 --- /dev/null +++ b/ecomm-demo/app/api/scrape/route.ts @@ -0,0 +1,66 @@ +import { NextRequest, NextResponse } from "next/server"; +import { ApifyClient } from "apify-client"; + +// Initialize the ApifyClient with API token from environment variables +const client = new ApifyClient({ + token: process.env.APIFY_TOKEN, +}); + +export async function POST(request: NextRequest) { + try { + const { query } = await request.json(); + + if (!query || typeof query !== "string") { + return NextResponse.json( + { error: "Query parameter is required" }, + { status: 400 } + ); + } + + if (!process.env.APIFY_TOKEN) { + return NextResponse.json( + { + error: "APIFY_TOKEN not configured. Please set up your Apify API token.", + details: "Get your token at https://console.apify.com/account#/integrations" + }, + { status: 500 } + ); + } + + // Call the E-commerce Scraping Tool Actor with keyword search + const run = await client.actor("apify/e-commerce-scraping-tool").call({ + keyword: query, + marketplaces: ["www.amazon.com"], // Default to Amazon for demo + maxProductResults: 20, // Limit to 20 products for demo + scrapeMode: "AUTO", + }); + + // Fetch the results from the Actor's default dataset + const { items } = await client.dataset(run.defaultDatasetId).listItems(); + + // Transform the data to match our Product type + const products = items.map((item: Record) => ({ + url: item.url || "", + title: item.name || "Unknown Product", + image: item.image || "", + description: item.description || "", + price: (item.offers as Record)?.price || 0, + })); + + return NextResponse.json({ + products, + runId: run.id, + runUrl: `https://console.apify.com/actors/runs/${run.id}` + }); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : "Unknown error occurred"; + console.error("Error calling Apify Actor:", error); + return NextResponse.json( + { + error: "Failed to scrape products", + details: errorMessage + }, + { status: 500 } + ); + } +} diff --git a/ecomm-demo/app/page.tsx b/ecomm-demo/app/page.tsx index 5561482..3cfe791 100644 --- a/ecomm-demo/app/page.tsx +++ b/ecomm-demo/app/page.tsx @@ -1,12 +1,65 @@ -import { products } from "@/data/products"; +"use client"; + +import { useState } from "react"; +import { products as mockProducts } from "@/data/products"; import { StatsCards } from "@/components/StatsCards"; import { ProductTable } from "@/components/ProductTable"; import { ProductCards } from "@/components/ProductCards"; import { SearchBar } from "@/components/SearchBar"; import { Separator } from "@/components/ui/separator"; import Image from "next/image"; +import type { Product, ScraperResponse, ScraperError } from "@/lib/types"; export default function Home() { + const [products, setProducts] = useState(mockProducts); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [dataSource, setDataSource] = useState<"Mock" | "Apify">("Mock"); + const [runUrl, setRunUrl] = useState(null); + + const handleSearch = async (query: string) => { + if (!query.trim()) { + return; + } + + setLoading(true); + setError(null); + setRunUrl(null); + + try { + const response = await fetch("/api/scrape", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ query }), + }); + + if (!response.ok) { + const errorData: ScraperError = await response.json(); + throw new Error(errorData.details || errorData.error || "Failed to fetch products"); + } + + const data: ScraperResponse = await response.json(); + + if (data.products && data.products.length > 0) { + setProducts(data.products); + setDataSource("Apify"); + if (data.runUrl) { + setRunUrl(data.runUrl); + } + } else { + setError("No products found for this search query. Try a different keyword."); + } + } catch (err) { + const errorMessage = err instanceof Error ? err.message : "An error occurred while fetching products"; + setError(errorMessage); + console.error("Search error:", err); + } finally { + setLoading(false); + } + }; + return (
{/* Header */} @@ -37,19 +90,54 @@ export default function Home() { Product Catalog Demo

- Showcase of e-commerce product data ready for Apify integration. - This demo displays scraped product information in multiple formats. + Search for products using Apify's E-Commerce Scraping Tool. + Enter a product keyword below to scrape real-time data from Amazon.

{/* Search Bar */}
- +
+ + {/* Loading State */} + {loading && ( +
+

+ 🔄 Scraping products... This may take a few moments. +

+
+ )} + + {/* Error State */} + {error && ( +
+

+ ⚠️ {error} +

+
+ )} + + {/* Run URL Link */} + {runUrl && ( +
+

+ ✓ Data scraped successfully!{" "} + + View run details in Apify Console + +

+
+ )}
{/* Stats Section */}
- +
diff --git a/ecomm-demo/components/ProductCards.tsx b/ecomm-demo/components/ProductCards.tsx index fb1517e..cdc7f09 100644 --- a/ecomm-demo/components/ProductCards.tsx +++ b/ecomm-demo/components/ProductCards.tsx @@ -10,11 +10,6 @@ interface ProductCardsProps { } export function ProductCards({ products }: ProductCardsProps) { - const truncate = (text: string, maxLength: number) => { - if (text.length <= maxLength) return text; - return text.substring(0, maxLength) + "..."; - }; - return (
diff --git a/ecomm-demo/components/StatsCards.tsx b/ecomm-demo/components/StatsCards.tsx index b37ae92..988fe48 100644 --- a/ecomm-demo/components/StatsCards.tsx +++ b/ecomm-demo/components/StatsCards.tsx @@ -1,11 +1,20 @@ -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Package } from "lucide-react"; +import { Product } from "@/lib/types"; interface StatsCardsProps { - productCount: number; + products: Product[]; + dataSource: "Mock" | "Apify"; } -export function StatsCards({ productCount }: StatsCardsProps) { +export function StatsCards({ products, dataSource }: StatsCardsProps) { + const productCount = products.length; + + // Calculate average price + const averagePrice = products.length > 0 + ? products.reduce((sum, product) => sum + product.price, 0) / products.length + : 0; + return (
@@ -27,9 +36,11 @@ export function StatsCards({ productCount }: StatsCardsProps) { $ -
-
+
+ {averagePrice > 0 ? `$${averagePrice.toFixed(2)}` : "-"} +

- Will be calculated from data + {averagePrice > 0 ? "Calculated from products" : "Will be calculated from data"}

@@ -37,12 +48,12 @@ export function StatsCards({ productCount }: StatsCardsProps) { Data Source - 🔗 + {dataSource === "Apify" ? "🚀" : "🔗"} -
Mock
+
{dataSource}

- Ready for Apify integration + {dataSource === "Apify" ? "Live scraped data" : "Ready for Apify integration"}

diff --git a/ecomm-demo/lib/types.ts b/ecomm-demo/lib/types.ts index acfc6da..65a4ac2 100644 --- a/ecomm-demo/lib/types.ts +++ b/ecomm-demo/lib/types.ts @@ -6,3 +6,14 @@ export type Product = { price: number; }; +export type ScraperResponse = { + products: Product[]; + runId?: string; + runUrl?: string; +}; + +export type ScraperError = { + error: string; + details?: string; +}; + diff --git a/ecomm-demo/next.config.ts b/ecomm-demo/next.config.ts index e9c0cb2..d1b28ea 100644 --- a/ecomm-demo/next.config.ts +++ b/ecomm-demo/next.config.ts @@ -11,6 +11,26 @@ const nextConfig: NextConfig = { protocol: "https", hostname: "apify.com", }, + { + protocol: "https", + hostname: "m.media-amazon.com", + }, + { + protocol: "https", + hostname: "images-na.ssl-images-amazon.com", + }, + { + protocol: "https", + hostname: "**.amazon.com", + }, + { + protocol: "https", + hostname: "i5.walmartimages.com", + }, + { + protocol: "https", + hostname: "**.walmartimages.com", + }, ], }, }; diff --git a/ecomm-demo/package.json b/ecomm-demo/package.json index 44871bf..3cef9a4 100644 --- a/ecomm-demo/package.json +++ b/ecomm-demo/package.json @@ -12,6 +12,7 @@ "@radix-ui/react-scroll-area": "^1.2.10", "@radix-ui/react-separator": "^1.1.8", "@radix-ui/react-slot": "^1.2.4", + "apify-client": "^2.19.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "lucide-react": "^0.553.0",