diff --git a/src/clis/tgb/post-comments.yaml b/src/clis/tgb/post-comments.yaml new file mode 100644 index 0000000..d595827 --- /dev/null +++ b/src/clis/tgb/post-comments.yaml @@ -0,0 +1,151 @@ +site: tgb +name: post-comments +description: 抓取淘股吧帖子分页评论 +domain: tgb.cn +strategy: cookie +browser: true + +args: + url: + type: str + required: true + description: 帖子URL（支持自动提取topicID和跳转页码） + page: + type: int + default: 1 + description: 起始页码，默认1 + max_pages: + type: int + default: 1 + description: 最大抓取页数，默认1 + limit: + type: int + default: 50 + description: 每页最大评论数 + +pipeline: + - navigate: ${{ args.url }} + - evaluate: | + (async () => { + // 从URL提取topicCode + const url = window.location.href; + const urlMatch = url.match(/\/a\/([a-zA-Z0-9]+)(?:-(\d+))?/); + if (!urlMatch) { + return { error: '无法从URL提取topicID' }; + } + + const topicCode = urlMatch[1]; + const currentPage = parseInt(urlMatch[2]) || 1; + const startPage = ${{ args.page }} || currentPage; + const maxPages = ${{ args.max_pages }} || 1; + const limitPerPage = ${{ args.limit }} || 50; + + const allComments = []; + let pagesScanned = 0; + + // 抓取多页评论 + for (let pageNum = startPage; pageNum < startPage + maxPages; pageNum++) { + const pageUrl = `https://www.tgb.cn/a/${topicCode}-${pageNum}?type=Z`; + + try { + const res = await fetch(pageUrl, { + credentials: 'include', + headers: { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'zh-CN,zh;q=0.9', + } + }); + + if (!res.ok) { + console.error(`Page ${pageNum} fetch failed: ${res.status}`); + continue; + } + + const html = await res.text(); + + if (!html.includes('gioMsg_R_')) { + console.error(`Page ${pageNum} has no comments`); + break; + } + + // 提取所有评论 + const commentRegex = /
]*subject=['"]([^'"]*)['"][^>]*userID=['"]([^'"]*)['"][^>]*userName=['"]([^'"]*)['"]\s*><\/div>/g; + let match; + const pageComments = []; + + while ((match = commentRegex.exec(html)) !== null) { + const commentId = match[1]; + const content = match[2] + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(/'/g, "'"); + const userId = match[3]; + const userName = match[4]; + + pageComments.push({ + comment_id: commentId, + user_id: userId, + user_name: userName, + content: content, + page: pageNum, + time: '', + floor: '', + parent_id: '' + }); + } + + // 提取每条评论的额外信息 + for (const comment of pageComments) { + const commentId = comment.comment_id; + const commentMarker = `id="gioMsg_R_${commentId}"`; + const idx = html.indexOf(commentMarker); + if (idx === -1) continue; + + // 提取评论块 + const nextCommentMatch = html.substring(idx + 1).match(/id="gioMsg_R_\d+"/); + let blockEnd; + if (nextCommentMatch) { + blockEnd = idx + 1 + html.substring(idx + 1).indexOf(nextCommentMatch[0]); + } else { + blockEnd = Math.min(html.length, idx + 5000); + } + const block = html.substring(idx, blockEnd); + + // 提取时间 + const timeMatch = block.match(/]*pcyclspan[^>]*>(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2})<\/span>/); + if (timeMatch) comment.time = timeMatch[1]; + + // 提取楼层 + const floorMatch = block.match(/>(沙发|板凳|地板|第\d+楼) { + // fetch 获取 HTML + const res = await fetch(window.location.href, { + credentials: 'include' + }); + const html = await res.text(); + + // 从 gioMsg 提取主题/标题（优先） + const subjectMatch = html.match(/id="gioMsg"[^>]*subject\s*=\s*"([^"]+)"/); + let title = subjectMatch ? subjectMatch[1].trim() : ''; + + // 备选：从 title 标签提取 + if (!title) { + const titleMatch = html.match(/([^<]+)<\/title>/); + if (titleMatch) { + title = titleMatch[1].replace(/_.*$/, '').trim(); + } + } + + // 从 gioMsg 提取作者信息 + const gioMatch = html.match(/id="gioMsg"[^>]*userName="([^"]+)"/); + const author = gioMatch ? gioMatch[1] : ''; + + // 提取时间 - 在 article-data 中 + const timeMatch = html.match(/(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2})/); + const publishTime = timeMatch ? timeMatch[1] : ''; + + // 提取浏览数 + const viewMatch = html.match(/浏览\s+(\d+)/); + const views = viewMatch ? viewMatch[1] : ''; + + // 提取评论数 + const commentMatch = html.match(/评论\s+(\d+)/); + const comments = commentMatch ? commentMatch[1] : ''; + + // 提取正文内容 - 在 article-text p_coten 中 + let contentHtml = ''; + + // 方法1: 匹配 article-text 到 videoImg + const contentMatch1 = html.match(/<div class="article-text[^"]*"[^>]*>([\s\S]*?)<div id="videoImg"/); + if (contentMatch1) { + contentHtml = contentMatch1[1]; + } else { + // 方法2: 匹配 article-text 到其闭合标签 + const contentMatch2 = html.match(/<div class="article-text[^"]*"[^>]*>([\s\S]*?)<\/div>\s*<div style="background:/); + if (contentMatch2) { + contentHtml = contentMatch2[1]; + } else { + // 方法3: 直接查找 article-text 的内容 + const contentMatch3 = html.match(/<div class="article-text[^"]*"[^>]*>([\s\S]*?)<\/div>/); + if (contentMatch3) { + contentHtml = contentMatch3[1]; + } + } + } + + // 清理 HTML 标签，提取纯文本段落 + const paragraphs = []; + if (contentHtml) { + // 将 <br> 转换为换行 + const text = contentHtml + .replace(/<br\s*\/?>/gi, '\n') + .replace(/<a[^>]*>/gi, '') // 移除链接标签开始 + .replace(/<\/a>/gi, '') // 移除链接标签结束 + .replace(/<[^>]+>/g, '') // 移除其他 HTML 标签 + .replace(/ /g, ' ') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"'); + + // 按空行分割段落 + const lines = text.split('\n'); + for (const line of lines) { + const para = line.trim(); + if (para && para.length > 0) { + paragraphs.push(para); + } + } + } + + // 生成 markdown + const mdLines = [ + `# ${title || '无标题'}`, + '', + `> **作者**: ${author || '未知'}`, + `> **发布时间**: ${publishTime || '未知'}`, + `> **浏览**: ${views || '-'} | **评论**: ${comments || '-'}`, + `> **原文**: ${window.location.href}`, + '', + '---', + '' + ]; + + for (const para of paragraphs) { + const clean = para.replace(/\s+/g, ' ').trim(); + if (clean) { + mdLines.push(clean); + mdLines.push(''); + } + } + + return { + title: title || '无标题', + author: author || '未知', + publish_time: publishTime || '未知', + views: views, + comments: comments, + url: window.location.href, + paragraphs: paragraphs.length, + markdown: mdLines.join('\n') + }; + })() + +columns: [title, author, publish_time, views, comments, paragraphs] diff --git a/src/clis/tgb/user-posts.yaml b/src/clis/tgb/user-posts.yaml new file mode 100644 index 0000000..fc9c0e9 --- /dev/null +++ b/src/clis/tgb/user-posts.yaml @@ -0,0 +1,123 @@ +site: tgb +name: user-posts +description: 获取用户文章列表 +domain: tgb.cn +strategy: cookie +browser: true + +args: + user_id: + type: str + required: true + description: 用户ID + limit: + type: int + default: 20 + description: 获取文章数量，默认20 + +pipeline: + - navigate: "https://www.tgb.cn/user/blog/moreTopic?userID=${{ args.user_id }}" + - evaluate: | + (async () => { + // 优先尝试从已加载的DOM中提取（更快） + const tables = document.querySelectorAll('table'); + const posts = []; + + for (const table of tables) { + const rows = table.querySelectorAll('tr'); + + for (const row of rows) { + const cells = row.querySelectorAll('td'); + if (cells.length < 8) continue; + + const titleCell = cells[1]; + if (!titleCell) continue; + + const link = titleCell.querySelector('a'); + if (!link) continue; + + const href = link.getAttribute('href') || ''; + if (!href.match(/^a\/[a-zA-Z0-9]+/)) continue; + + const url = 'https://www.tgb.cn/' + href; + const title = titleCell.innerText.trim(); + const author = cells[2]?.innerText?.trim() || ''; + const commentTime = cells[3]?.innerText?.trim() || ''; + const publishDate = cells[7]?.innerText?.trim() || ''; + + if (title && url) { + posts.push({ + title: title, + author: author, + comment_time: commentTime, + publish_date: publishDate, + url: url + }); + } + } + } + + // 如果DOM提取不到，尝试fetch获取HTML + if (posts.length === 0) { + try { + const res = await fetch(window.location.href, { + credentials: 'include' + }); + const html = await res.text(); + + // 解析表格行 + const rowRegex = /<tr[^>]*>([\s\S]*?)<\/tr>/g; + let rowMatch; + + while ((rowMatch = rowRegex.exec(html)) !== null) { + const rowHtml = rowMatch[1]; + + // 提取所有单元格 + const cellRegex = /<td[^>]*>([\s\S]*?)<\/td>/g; + const cells = []; + let cellMatch; + while ((cellMatch = cellRegex.exec(rowHtml)) !== null) { + cells.push(cellMatch[1]); + } + + if (cells.length < 8) continue; + + // 提取标题和链接 + const titleMatch = cells[1].match(/<a[^>]*href=["\'](a\/[a-zA-Z0-9]+)["\'][^>]*>([^<]+)/); + if (!titleMatch) continue; + + const href = titleMatch[1]; + const title = titleMatch[2].trim(); + + // 提取其他字段 + const author = cells[2].replace(/<[^>]+>/g, '').trim(); + const commentTime = cells[3].replace(/<[^>]+>/g, '').trim(); + const publishDate = cells[7].replace(/<[^>]+>/g, '').trim(); + + if (title && href) { + posts.push({ + title: title, + author: author, + comment_time: commentTime, + publish_date: publishDate, + url: 'https://www.tgb.cn/' + href + }); + } + } + } catch (e) { + // fetch失败，返回空数组 + } + } + + // 去重 + const seen = new Set(); + const unique = posts.filter(p => { + if (seen.has(p.url)) return false; + seen.add(p.url); + return true; + }); + + return unique.slice(0, ${{ args.limit }}); + })() + +columns: [title, author, comment_time, publish_date, url]