diff --git a/packages/fancy-sitemap/README.md b/packages/fancy-sitemap/README.md
new file mode 100644
index 00000000..64126270
--- /dev/null
+++ b/packages/fancy-sitemap/README.md
@@ -0,0 +1,2 @@
+
+maxCacheDays - Only works when {path} is provided, so we check if there is a sitemap.xml already created in your filesystem.
\ No newline at end of file
diff --git a/packages/fancy-sitemap/index.js b/packages/fancy-sitemap/index.js
new file mode 100644
index 00000000..170038e6
--- /dev/null
+++ b/packages/fancy-sitemap/index.js
@@ -0,0 +1,3 @@
+import lib from './lib'
+
+module.exports = lib
\ No newline at end of file
diff --git a/packages/fancy-sitemap/lib/index.js b/packages/fancy-sitemap/lib/index.js
new file mode 100644
index 00000000..9b748794
--- /dev/null
+++ b/packages/fancy-sitemap/lib/index.js
@@ -0,0 +1,203 @@
+import fs from 'fs';
+import util from 'util';
+import moment from 'moment';
+
+const SitemapCrawler = require('sitemap-generator');
+const Generator = require('sitemap')
+
+const CHANGE_FREQ_ALWAYS = 'always'
+const CHANGE_FREQ_HOURLY = 'hourly'
+const CHANGE_FREQ_DAILY = 'daily'
+const CHANGE_FREQ_WEEKLY = 'weekly'
+const CHANGE_FREQ_MONTHLY = 'monthly'
+const CHANGE_FREQ_YEARLY = 'yearly'
+const CHANGE_FREQ_NEVER = 'never'
+
+const DEFAULT_CHANGE_FREQ = CHANGE_FREQ_MONTHLY
+const DEFAULT_PRIORITY = 0.5
+
+const CACHE_TIME = 600000
+
+/**
+ * Gets all the URLS that a website has accessible by users.
+ *
+ * @param {*} options
+ */
+const runCrawler = (options) => {
+ let urls = [];
+ let errors = [];
+
+ const {hostname} = options;
+
+ // NB: We use SitemapCrawler generator to create all the sitemap tree.
+ // filepath should be null, so it doesn't generate an xml file.
+ const crawlerOptions = {
+ stripQuerystring: options.stripQuerystring || true,
+ filepath: null,
+ }
+
+ const crawler = SitemapCrawler(hostname, crawlerOptions);
+
+ return new Promise((resolve, reject) => {
+ crawler.start();
+
+ crawler.on('add', (url) => {
+ urls.push(url)
+ });
+
+ crawler.on('error', (error) => {
+ errors.push(error)
+ });
+
+ crawler.on('done', () => {
+ resolve([urls, errors])
+ });
+ })
+}
+
+const getXmlUrls = (options, generatedUrls = []) => {
+ const {rules = [], defaultPriority, defaultChangeFreq} = options;
+
+ const urls = generatedUrls.map((url) => {
+ const foundRule = rules.find(({path}) => (
+ RegExp(path,'g').exec(url) !== null
+ )) || {}
+
+ // TODO: Once we check the validity of the values and set the correct ones
+ // we can remove the last condition ||
+ return {
+ url,
+ changefreq: foundRule.changeFreq || defaultChangeFreq || DEFAULT_CHANGE_FREQ,
+ priority: foundRule.priority || defaultPriority || DEFAULT_PRIORITY,
+ }
+ })
+
+ return urls
+}
+
+const generateXmlSitemap = (options, generatedUrls = []) => {
+ const urls = getXmlUrls(options, generatedUrls)
+ const sitemapGenerator = Generator.createSitemap ({
+ cacheTime: CACHE_TIME, // 600 sec - cache purge period
+ urls,
+ })
+
+ return new Promise((resolve, reject) => {
+ sitemapGenerator.toXML((err, xml) => {
+ if (err) {
+ return reject(err)
+ }
+
+ return resolve(xml)
+ })
+ })
+}
+
+const createSitemapFile = (path, fileContent) => (
+ new Promise((resolve, reject) => (
+ fs.writeFile(path, fileContent, (err) => {
+ if (err) {
+ return reject(err);
+ }
+ return resolve();
+ })
+ ))
+)
+
+const shouldReturnCachedFile = (path, maxCacheDays) => (
+ new Promise((resolve, reject) => (
+ fs.stat(path, (err, stats) => {
+ if (err) {
+ return resolve(false)
+ }
+
+ const today = moment(new Date(), 'YYYY-MM-DD')
+ const fileModifiedDate = moment(new Date(util.inspect(stats.mtime)), 'YYYY-MM-DD')
+ const creationDays = moment.duration(today.diff(fileModifiedDate)).asDays()
+
+ return resolve(parseInt(creationDays) < maxCacheDays)
+ })
+ ))
+)
+
+const getFileXmlSitemap = (path) => (
+ new Promise((resolve, reject) => (
+ fs.readFile(path, (error, fileContent) => {
+ if (error) {
+ return reject(error)
+ }
+
+ return resolve(fileContent)
+ })
+ ))
+)
+
+const start = (options) => (
+ // TODO: Check validity of the options object.
+ // Example: hostname is provided and required!
+
+ // TODO: Also check if user default values are valid:
+ // priority should be [0-1]
+ // changeFreq should be one of the constants above
+ // otherwise, sets the default values: DEFAULT_CHANGE_FREQ and DEFAULT_PRIORITY
+
+ new Promise(async (resolve, reject) => {
+ const {maxCacheDays} = options;
+
+ // TODO: Clean this logic... Probably put it inside a method?
+ if (maxCacheDays && maxCacheDays > 0) {
+ const {path} = options;
+
+ try {
+ const hasCachedFile = await shouldReturnCachedFile(path, maxCacheDays)
+
+ if (hasCachedFile) {
+ const fileXmlSitemap = await getFileXmlSitemap(path);
+ return resolve(fileXmlSitemap);
+ }
+
+ // SKIP Cached version: Generate the file again
+ } catch (error) {
+ console.log('Trying to get file from cached failed...')
+ console.log('error', error)
+ // SKIP Cached version: Generate the file again
+ }
+ }
+
+ // Generate file...
+ let urls;
+ let errors;
+
+ try {
+ [urls, errors] = await runCrawler(options)
+ } catch (error) {
+ return reject(error);
+ }
+
+ // TODO: Do something with the errors???
+
+ try {
+ const generatedXmlSitemap = await generateXmlSitemap(options, urls);
+ const {path} = options;
+
+ if (path) {
+ // NB: If it returns an error, we don't care.
+ await createSitemapFile(path, generatedXmlSitemap);
+
+ return resolve(generatedXmlSitemap);
+ } else {
+ return resolve(generatedXmlSitemap);
+ }
+ }
+ catch (error) {
+ console.log(error)
+ return reject(error);
+ }
+ })
+)
+
+const SitemapGenerator = {
+ start,
+}
+
+module.exports = SitemapGenerator
\ No newline at end of file
diff --git a/packages/fancy-sitemap/package.json b/packages/fancy-sitemap/package.json
new file mode 100644
index 00000000..8e6db23e
--- /dev/null
+++ b/packages/fancy-sitemap/package.json
@@ -0,0 +1,23 @@
+{
+ "name": "fancy-sitemap",
+ "description": "PUT A FANCY DESCRIPTION",
+ "homepage": "http://www.github.com/ferreiro/fancy-sitemap",
+ "version": "0.0.9",
+ "private": false,
+ "license": "MIT",
+ "keywords": [],
+ "author": {
+ "name": "Jorge Ferreiro",
+ "email": "jorge@ferreiro.me",
+ "url": "https://www.ferreiro.me/"
+ },
+ "scripts": {
+ "start": ""
+ },
+ "dependencies": {
+ "lodash": "4.17.11",
+ "sitemap": "2.1.0",
+ "sitemap-generator": "8.3.3"
+ },
+ "devDependencies": {}
+}
diff --git a/packages/ferreiro-server/env/development.js b/packages/ferreiro-server/env/development.js
index f58d150c..3ba707af 100644
--- a/packages/ferreiro-server/env/development.js
+++ b/packages/ferreiro-server/env/development.js
@@ -5,6 +5,7 @@ module.exports = {
ADMIN_EMAIL: 'admin',
ADMIN_PASS: 'admin',
MAILCHIMP_API_TOKEN: process.env.MAILCHIMP_API_TOKEN,
+ MAX_CACHE_DAYS_SITEMAP: 0,
// AMAZON WEB SERVICES
S3_REGION: process.env.S3_REGION,
diff --git a/packages/ferreiro-server/env/production.js b/packages/ferreiro-server/env/production.js
index 06f77151..61b83f3e 100644
--- a/packages/ferreiro-server/env/production.js
+++ b/packages/ferreiro-server/env/production.js
@@ -5,6 +5,7 @@ module.exports = {
ADMIN_EMAIL: process.env.ADMIN_EMAIL,
ADMIN_PASS: process.env.ADMIN_PASS,
MAILCHIMP_API_TOKEN: process.env.MAILCHIMP_API_TOKEN,
+ MAX_CACHE_DAYS_SITEMAP: process.env.MAX_CACHE_DAYS_SITEMAP,
// AMAZON WEB SERVICES
S3_REGION: process.env.S3_REGION,
diff --git a/packages/ferreiro-server/package.json b/packages/ferreiro-server/package.json
index ed0d3578..2f8cd0dc 100644
--- a/packages/ferreiro-server/package.json
+++ b/packages/ferreiro-server/package.json
@@ -48,6 +48,7 @@
"cookie-parser": "^1.4.4",
"debug": "^3.1.0",
"express": "^4.16.1",
+ "express-rate-limit": "3.5.0",
"express-recaptcha": "^3.0.0",
"express-session": "^1.15.6",
"force-ssl-heroku": "^1.0.2",
@@ -56,6 +57,7 @@
"lodash": "^4.17.11",
"mailchimp-api-v3": "^1.7.1",
"marked": "^0.6.1",
+ "moment": "2.24.0",
"mongoose": "^4.11.13",
"mongoose-paginate": "^5.0.3",
"mongoose-permalink": "^2.0.0",
diff --git a/packages/ferreiro-server/pene.jpg b/packages/ferreiro-server/pene.jpg
new file mode 100644
index 00000000..3c96dee6
Binary files /dev/null and b/packages/ferreiro-server/pene.jpg differ
diff --git a/packages/ferreiro-server/setup/setupMiddlewares.js b/packages/ferreiro-server/setup/setupMiddlewares.js
index b416451c..ecd7f5a5 100644
--- a/packages/ferreiro-server/setup/setupMiddlewares.js
+++ b/packages/ferreiro-server/setup/setupMiddlewares.js
@@ -7,6 +7,15 @@ const helmet = require('helmet')
const session = require('express-session')
const express = require('express')
const compression = require('compression')
+const rateLimit = require('express-rate-limit');
+
+const crawlersRateLimit = rateLimit({
+ windowMs: 15 * 60 * 1000, // 15 minutes
+ max: 15 // limit each IP to 15 requests per windowMs
+});
+
+// TODO: Move to it's own repository and deploy...
+const sitemapGenerator = require('../../fancy-sitemap/index')
const env = require('../env')
@@ -18,11 +27,56 @@ module.exports = (app) => {
app.use(compression())
// Search engines
- app.get('/robots.txt', (req, res) => {
+ app.get('/robots.txt', crawlersRateLimit, (req, res) => {
res.sendFile(path.join(__dirname + '/../robots.txt'))
})
- app.get('/sitemap.xml', (req, res) => {
- res.sendFile(path.join(__dirname + '/../sitemap.xml'))
+
+ app.get('/sitemap.xml', crawlersRateLimit, (req, res) => {
+ // TODO: Move the sitemap configuration to it's own file...
+ sitemapGenerator
+ .start({
+ hostname: process.env.NODE_ENV === 'DEV' ? 'localhost:3000' : 'https://www.ferreiro.me',
+ path: 'sitemap.xml',
+ maxCacheDays: env.MAX_CACHE_DAYS_SITEMAP || 0,
+ stripQuerystring: true,
+ filepath: 'sitemap.xml',
+ defaultPriority: 0.8,
+ defaultChangeFreq: 'yearly',
+ rules: [
+ {
+ path: '/$',
+ changeFreq: 'monthly',
+ priority: 1,
+ },
+ {
+ path: '/about$',
+ changeFreq: 'monthly',
+ priority: 1,
+ },
+ {
+ path: '/blog$',
+ changeFreq: 'weekly',
+ priority: 1,
+ },
+ {
+ path: '/blog/*',
+ changeFreq: 'monthly',
+ priority: 0.9,
+ },
+ {
+ path: '/talks$',
+ changeFreq: 'monthly',
+ priority: 1,
+ },
+ ],
+ })
+ .then((sitemap) => {
+ res.header('Content-Type', 'application/xml');
+ return res.send(sitemap)
+ })
+ .catch((error) => (
+ res.status(500).send(error).end()
+ ))
})
// Serve static bower: http://goo.gl/e2nTBf
diff --git a/packages/ferreiro-server/sitemap.xml b/packages/ferreiro-server/sitemap.xml
index de4da69d..6dc9eb79 100644
--- a/packages/ferreiro-server/sitemap.xml
+++ b/packages/ferreiro-server/sitemap.xml
@@ -1,63 +1 @@
-
-
-
-
- https://www.ferreiro.me/
- weekly
- 1.00
-
-
- https://www.ferreiro.me/about
- monthly
- 0.92
-
-
- https://www.ferreiro.me/blog
- weekly
- 1
-
-
- https://www.ferreiro.me/blog/interview-with-cesar-puerta-staff-software
- monthly
- 1
-
-
- https://www.ferreiro.me/blog/jorge-ferreiro-joins-eventbrite-as-software-engineer
- weekly
- 1
-
-
- https://www.ferreiro.me/blog/speed-up-your-website-frontend-8-practical-tips
- weekly
- 1
-
-
- https://www.ferreiro.me/blog/welcome-to-my-new-digital-space-welcome-to-jorge-ferreiro-blog
- weekly
- 1
-
-
- https://www.ferreiro.me/portfolio
- monthly
- 0.92
-
-
- https://www.ferreiro.me/talks
- monthly
- 0.95
-
-
- https://www.ferreiro.me/resume/jorge_ferreiro_resume.pdf
- monthly
- 0.9
-
-
- https://www.ferreiro.me/contact
- never
- 0.5
-
-
\ No newline at end of file
+https://www.ferreiro.me/monthly1.0https://www.ferreiro.me/aboutmonthly1.0https://www.ferreiro.me/blogmonthly1.0https://www.ferreiro.me/talksmonthly1.0https://www.ferreiro.me/portfolioyearly0.8https://www.ferreiro.me/contactyearly0.8https://www.ferreiro.me/blog/interview-with-cesar-puerta-staff-softwaremonthly0.9https://www.ferreiro.me/blog/crushing-on-site-interviewsmonthly0.9https://www.ferreiro.me/blog/jorge-ferreiro-joins-eventbrite-as-software-engineermonthly0.9https://www.ferreiro.me/newsletteryearly0.8https://www.ferreiro.me/blog/speed-up-your-website-frontend-8-practical-tipsmonthly0.9https://www.ferreiro.me/resume/jorge_ferreiro_resume.pdfyearly0.8https://www.ferreiro.me/blog/welcome-to-my-new-digital-space-welcome-to-jorge-ferreiro-blogmonthly0.9https://www.ferreiro.me/contact/talkyearly0.8https://www.ferreiro.me/blog/category/softwaremonthly0.9https://www.ferreiro.me/contact/feedbackyearly0.8https://www.ferreiro.me/blog/category/productmonthly0.9https://www.ferreiro.me/blog/category/entrepreneurshipmonthly0.9
\ No newline at end of file
diff --git a/packages/ferreiro-server/web/content/english/talks.json b/packages/ferreiro-server/web/content/english/talks.json
index bf1c8c47..3cdd3660 100644
--- a/packages/ferreiro-server/web/content/english/talks.json
+++ b/packages/ferreiro-server/web/content/english/talks.json
@@ -9,6 +9,30 @@
}
},
"pastTalks": [
+ {
+ "title": "Github Like a Pro by Jorge Ferreiro - Adalab bootcamp #adalabGithub @Google Campus Madrid",
+ "summary": "Why I ❤️ Github? What makes your Github profile stands out? Roast my Github. How do I use Github?",
+ "permalink": "https://speakerdeck.com/ferreiro/github-like-a-pro-jorge-ferreiro-adalab-bootcamp-at-google-campus-madrid",
+ "type": "slides",
+ "pic": "/images/talks/jorge_ferreiro_creating_github_profile_like_pro.jpg",
+ "language": {
+ "display": "English",
+ "icon": "icon-english"
+ },
+ "date": "May, 7th 2019",
+ "location": {
+ "venue": "London",
+ "city": "Madrid"
+ },
+ "event": "",
+ "buttons": [
+ {
+ "title": "Read the slides",
+ "type": "slides",
+ "url": "https://speakerdeck.com/ferreiro/github-like-a-pro-jorge-ferreiro-adalab-bootcamp-at-google-campus-madrid"
+ }
+ ]
+ },
{
"title": "Web Perfomance: Expectations Vs Reality - JSRoundabout by Jorge Ferreiro - #TwitterPerf",
"summary": "“Why is my website slow?”, “How can I improve the loading time?”, “How does the browser render a website?”, “What techniques can I apply to boost the speed of my website?”.",
diff --git a/packages/ferreiro-server/web/public/src/images/talks/jorge_ferreiro_creating_github_profile_like_pro.jpg b/packages/ferreiro-server/web/public/src/images/talks/jorge_ferreiro_creating_github_profile_like_pro.jpg
new file mode 100644
index 00000000..7d374139
Binary files /dev/null and b/packages/ferreiro-server/web/public/src/images/talks/jorge_ferreiro_creating_github_profile_like_pro.jpg differ
diff --git a/packages/ferreiro-server/web/public/src/images/talks/jorge_ferreiro_creating_github_profile_like_pro.psd b/packages/ferreiro-server/web/public/src/images/talks/jorge_ferreiro_creating_github_profile_like_pro.psd
new file mode 100644
index 00000000..cc0748c8
Binary files /dev/null and b/packages/ferreiro-server/web/public/src/images/talks/jorge_ferreiro_creating_github_profile_like_pro.psd differ
diff --git a/yarn.lock b/yarn.lock
index 345e6491..4861f655 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -12,6 +12,10 @@
dependencies:
"@types/babel-types" "*"
+"@types/node@*":
+ version "11.13.8"
+ resolved "https://registry.yarnpkg.com/@types/node/-/node-11.13.8.tgz#e5d71173c95533be9842b2c798978f095f912aab"
+
abbrev@1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
@@ -286,6 +290,18 @@ async@2.6.0:
dependencies:
lodash "^4.14.0"
+async@2.6.1:
+ version "2.6.1"
+ resolved "https://registry.yarnpkg.com/async/-/async-2.6.1.tgz#b245a23ca71930044ec53fa46aa00a3e87c6a610"
+ dependencies:
+ lodash "^4.17.10"
+
+async@^2.1.4:
+ version "2.6.2"
+ resolved "https://registry.yarnpkg.com/async/-/async-2.6.2.tgz#18330ea7e6e313887f5d2f2a904bac6fe4dd5381"
+ dependencies:
+ lodash "^4.17.11"
+
asynckit@^0.4.0:
version "0.4.0"
resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
@@ -1002,6 +1018,10 @@ body-parser@1.18.3, body-parser@^1.18.2:
raw-body "2.3.3"
type-is "~1.6.16"
+boolbase@~1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e"
+
bower@^1.8.8:
version "1.8.8"
resolved "https://registry.yarnpkg.com/bower/-/bower-1.8.8.tgz#82544be34a33aeae7efb8bdf9905247b2cffa985"
@@ -1287,6 +1307,17 @@ character-parser@^2.1.1:
dependencies:
is-regex "^1.0.3"
+cheerio@1.0.0-rc.2:
+ version "1.0.0-rc.2"
+ resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-1.0.0-rc.2.tgz#4b9f53a81b27e4d5dac31c0ffd0cfa03cc6830db"
+ dependencies:
+ css-select "~1.2.0"
+ dom-serializer "~0.1.0"
+ entities "~1.1.1"
+ htmlparser2 "^3.9.1"
+ lodash "^4.15.0"
+ parse5 "^3.0.1"
+
chokidar@^1.6.1:
version "1.7.0"
resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-1.7.0.tgz#798e689778151c8076b4b360e5edd28cda2bb468"
@@ -1652,6 +1683,16 @@ core-util-is@1.0.2, core-util-is@~1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
+cp-file@6.0.0:
+ version "6.0.0"
+ resolved "https://registry.yarnpkg.com/cp-file/-/cp-file-6.0.0.tgz#f38477ece100b403fcf780fd34d030486beb693e"
+ dependencies:
+ graceful-fs "^4.1.2"
+ make-dir "^1.0.0"
+ nested-error-stacks "^2.0.0"
+ pify "^3.0.0"
+ safe-buffer "^5.0.1"
+
crc@3.4.4:
version "3.4.4"
resolved "https://registry.yarnpkg.com/crc/-/crc-3.4.4.tgz#9da1e980e3bd44fc5c93bf5ab3da3378d85e466b"
@@ -1694,7 +1735,7 @@ cross-spawn@^6.0.0:
shebang-command "^1.2.0"
which "^1.2.9"
-crypto-random-string@^1.0.0:
+crypto-random-string@1.0.0, crypto-random-string@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-1.0.0.tgz#a230f64f568310e1498009940790ec99545bca7e"
@@ -1702,6 +1743,19 @@ css-color-names@0.0.4:
version "0.0.4"
resolved "http://registry.npmjs.org/css-color-names/-/css-color-names-0.0.4.tgz#808adc2e79cf84738069b646cb20ec27beb629e0"
+css-select@~1.2.0:
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/css-select/-/css-select-1.2.0.tgz#2b3a110539c5355f1cd8d314623e870b121ec858"
+ dependencies:
+ boolbase "~1.0.0"
+ css-what "2.1"
+ domutils "1.5.1"
+ nth-check "~1.0.1"
+
+css-what@2.1:
+ version "2.1.3"
+ resolved "https://registry.yarnpkg.com/css-what/-/css-what-2.1.3.tgz#a6d7604573365fe74686c3f311c56513d88285f2"
+
cssnano@^3.0.0:
version "3.10.0"
resolved "http://registry.npmjs.org/cssnano/-/cssnano-3.10.0.tgz#4f38f6cea2b9b17fa01490f23f1dc68ea65c1c38"
@@ -1762,6 +1816,10 @@ dasherize@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/dasherize/-/dasherize-2.0.0.tgz#6d809c9cd0cf7bb8952d80fc84fa13d47ddb1308"
+date-fns@1.29.0:
+ version "1.29.0"
+ resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-1.29.0.tgz#12e609cdcb935127311d04d33334e2960a2a54e6"
+
date-fns@^1.23.0:
version "1.30.1"
resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-1.30.1.tgz#2e71bf0b119153dbb4cc4e88d9ea5acfb50dc05c"
@@ -1861,7 +1919,7 @@ deep-extend@^0.6.0:
version "0.6.0"
resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac"
-defaults@^1.0.0:
+defaults@^1.0.0, defaults@^1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/defaults/-/defaults-1.0.3.tgz#c656051e9817d9ff08ed881477f3fe4019f3ef7d"
dependencies:
@@ -1991,7 +2049,14 @@ dom-serializer@0:
domelementtype "~1.1.1"
entities "~1.1.1"
-domelementtype@1, domelementtype@^1.3.0:
+dom-serializer@~0.1.0:
+ version "0.1.1"
+ resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.1.tgz#1ec4059e284babed36eec2941d4a970a189ce7c0"
+ dependencies:
+ domelementtype "^1.3.0"
+ entities "^1.1.1"
+
+domelementtype@1, domelementtype@^1.3.0, domelementtype@^1.3.1:
version "1.3.1"
resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f"
@@ -2005,6 +2070,13 @@ domhandler@^2.3.0:
dependencies:
domelementtype "1"
+domutils@1.5.1:
+ version "1.5.1"
+ resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.5.1.tgz#dcd8488a26f563d61079e48c9f7b7e32373682cf"
+ dependencies:
+ dom-serializer "0"
+ domelementtype "1"
+
domutils@^1.5.1:
version "1.7.0"
resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.7.0.tgz#56ea341e834e06e6748af7a1cb25da67ea9f8c2a"
@@ -2271,6 +2343,12 @@ expect-ct@0.1.1:
version "0.1.1"
resolved "https://registry.yarnpkg.com/expect-ct/-/expect-ct-0.1.1.tgz#de84476a2dbcb85000d5903737e9bc8a5ba7b897"
+express-rate-limit@3.5.0:
+ version "3.5.0"
+ resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-3.5.0.tgz#6aa3f62f5adc4dbea9162e8101208d7aa5745789"
+ dependencies:
+ defaults "^1.0.3"
+
express-recaptcha@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/express-recaptcha/-/express-recaptcha-3.0.1.tgz#6ff6b6e6ed43e6ef78643f56e3aa8ffe280394c8"
@@ -3288,6 +3366,17 @@ htmlparser2@^3.10.0:
inherits "^2.0.1"
readable-stream "^3.0.6"
+htmlparser2@^3.9.1:
+ version "3.10.1"
+ resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.10.1.tgz#bd679dc3f59897b6a34bb10749c855bb53a9392f"
+ dependencies:
+ domelementtype "^1.3.1"
+ domhandler "^2.3.0"
+ domutils "^1.5.1"
+ entities "^1.1.1"
+ inherits "^2.0.1"
+ readable-stream "^3.1.1"
+
http-errors@1.6.3, http-errors@~1.6.2, http-errors@~1.6.3:
version "1.6.3"
resolved "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz#8b55680bb4be283a0b5bf4ea2e38580be1d9320d"
@@ -3326,7 +3415,7 @@ iconv-lite@0.4.23:
dependencies:
safer-buffer ">= 2.1.2 < 3"
-iconv-lite@^0.4.4:
+iconv-lite@^0.4.13, iconv-lite@^0.4.4:
version "0.4.24"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
dependencies:
@@ -4323,7 +4412,7 @@ lodash.values@~2.4.1:
dependencies:
lodash.keys "~2.4.1"
-"lodash@4.6.1 || ^4.16.1", lodash@^4.0.0, lodash@^4.13.1, lodash@^4.14.0, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.4, lodash@^4.3.0, lodash@^4.5.1, lodash@^4.7.0, lodash@~4.17.10:
+lodash@4.17.11, "lodash@4.6.1 || ^4.16.1", lodash@^4.0.0, lodash@^4.13.1, lodash@^4.14.0, lodash@^4.15.0, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.4, lodash@^4.3.0, lodash@^4.5.1, lodash@^4.7.0, lodash@~4.17.10:
version "4.17.11"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
@@ -4594,6 +4683,10 @@ minizlib@^1.1.1:
dependencies:
minipass "^2.2.1"
+mitt@1.1.3:
+ version "1.1.3"
+ resolved "https://registry.yarnpkg.com/mitt/-/mitt-1.1.3.tgz#528c506238a05dce11cd914a741ea2cc332da9b8"
+
mixin-deep@^1.2.0:
version "1.3.1"
resolved "https://registry.yarnpkg.com/mixin-deep/-/mixin-deep-1.3.1.tgz#a49e7268dce1a0d9698e45326c5626df3543d0fe"
@@ -4607,6 +4700,10 @@ mixin-deep@^1.2.0:
dependencies:
minimist "0.0.8"
+moment@2.24.0:
+ version "2.24.0"
+ resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b"
+
mongodb-core@2.1.18:
version "2.1.18"
resolved "https://registry.yarnpkg.com/mongodb-core/-/mongodb-core-2.1.18.tgz#4c46139bdf3a1f032ded91db49f38eec01659050"
@@ -4769,6 +4866,10 @@ negotiator@0.6.1:
version "0.6.1"
resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9"
+nested-error-stacks@^2.0.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/nested-error-stacks/-/nested-error-stacks-2.1.0.tgz#0fbdcf3e13fe4994781280524f8b96b0cdff9c61"
+
nice-try@^1.0.4:
version "1.0.5"
resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
@@ -4970,6 +5071,10 @@ normalize-range@^0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/normalize-range/-/normalize-range-0.1.2.tgz#2d10c06bdfd312ea9777695a4d28439456b75942"
+normalize-url@3.3.0:
+ version "3.3.0"
+ resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-3.3.0.tgz#b2e1c4dc4f7c6d57743df733a4f5978d18650559"
+
normalize-url@^1.4.0:
version "1.9.1"
resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-1.9.1.tgz#2cc0d66b31ea23036458436e3620d85954c66c3c"
@@ -5042,6 +5147,12 @@ npm-run-path@^2.0.0:
gauge "~2.7.3"
set-blocking "~2.0.0"
+nth-check@~1.0.1:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-1.0.2.tgz#b2bd295c37e3dd58a3bf0700376663ba4d9cf05c"
+ dependencies:
+ boolbase "~1.0.0"
+
num2fraction@^1.2.2:
version "1.2.2"
resolved "https://registry.yarnpkg.com/num2fraction/-/num2fraction-1.2.2.tgz#6f682b6a027a4e9ddfa4564cd2589d1d4e669ede"
@@ -5334,6 +5445,12 @@ parse-passwd@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/parse-passwd/-/parse-passwd-1.0.0.tgz#6d5b934a456993b23d37f40a382d6f1666a8e5c6"
+parse5@^3.0.1:
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/parse5/-/parse5-3.0.3.tgz#042f792ffdd36851551cf4e9e066b3874ab45b5c"
+ dependencies:
+ "@types/node" "*"
+
parseurl@~1.3.2:
version "1.3.2"
resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.2.tgz#fc289d4ed8993119460c156253262cdc8de65bf3"
@@ -5917,6 +6034,10 @@ querystring@0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
+querystringify@^2.0.0:
+ version "2.1.1"
+ resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.1.1.tgz#60e5a5fd64a7f8bfa4d2ab2ed6fdf4c85bad154e"
+
random-bytes@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/random-bytes/-/random-bytes-1.0.0.tgz#4f68a1dc0ae58bd3fb95848c30324db75d64360b"
@@ -6055,6 +6176,14 @@ readable-stream@^3.0.6:
string_decoder "^1.1.1"
util-deprecate "^1.0.1"
+readable-stream@^3.1.1:
+ version "3.3.0"
+ resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.3.0.tgz#cb8011aad002eb717bf040291feba8569c986fb9"
+ dependencies:
+ inherits "^2.0.3"
+ string_decoder "^1.1.1"
+ util-deprecate "^1.0.1"
+
readdirp@^2.0.0, readdirp@^2.2.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-2.2.1.tgz#0e87622a3325aa33e892285caf8b4e846529a525"
@@ -6244,6 +6373,10 @@ require_optional@~1.0.0:
resolve-from "^2.0.0"
semver "^5.1.0"
+requires-port@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff"
+
resolve-dir@^1.0.0, resolve-dir@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/resolve-dir/-/resolve-dir-1.0.1.tgz#79a40644c362be82f26effe739c9bb5382046f43"
@@ -6288,6 +6421,10 @@ rimraf@2, rimraf@^2.2.6, rimraf@^2.2.8, rimraf@^2.5.1, rimraf@^2.5.4, rimraf@^2.
dependencies:
glob "^7.0.5"
+robots-parser@^1.0.0:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/robots-parser/-/robots-parser-1.0.2.tgz#9ebe25b1a2c52773cbe6f1dbe90ebc9518089009"
+
run-async@^0.1.0:
version "0.1.0"
resolved "https://registry.yarnpkg.com/run-async/-/run-async-0.1.0.tgz#c8ad4a5e110661e402a7d21b530e009f25f8e389"
@@ -6495,6 +6632,38 @@ signal-exit@^3.0.0, signal-exit@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d"
+simplecrawler@1.1.6:
+ version "1.1.6"
+ resolved "https://registry.yarnpkg.com/simplecrawler/-/simplecrawler-1.1.6.tgz#06a0f96400e9108d9b86f2e2b81cfb69b9130cbb"
+ dependencies:
+ async "^2.1.4"
+ iconv-lite "^0.4.13"
+ robots-parser "^1.0.0"
+ urijs "^1.18.11"
+
+sitemap-generator@8.3.3:
+ version "8.3.3"
+ resolved "https://registry.yarnpkg.com/sitemap-generator/-/sitemap-generator-8.3.3.tgz#37cfeb86b25ab90a0c96e83cbde596ff911fbed4"
+ dependencies:
+ async "2.6.1"
+ cheerio "1.0.0-rc.2"
+ cp-file "6.0.0"
+ crypto-random-string "1.0.0"
+ date-fns "1.29.0"
+ lodash "4.17.11"
+ mitt "1.1.3"
+ normalize-url "3.3.0"
+ simplecrawler "1.1.6"
+ url-parse "1.4.3"
+
+sitemap@2.1.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-2.1.0.tgz#1633cb88c196d755ad94becfb1c1bcacc6d3425a"
+ dependencies:
+ lodash "^4.17.10"
+ url-join "^4.0.0"
+ xmlbuilder "^10.0.0"
+
slash@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55"
@@ -7250,16 +7419,31 @@ uri-js@^4.2.2:
dependencies:
punycode "^2.1.0"
+urijs@^1.18.11:
+ version "1.19.1"
+ resolved "https://registry.yarnpkg.com/urijs/-/urijs-1.19.1.tgz#5b0ff530c0cbde8386f6342235ba5ca6e995d25a"
+
urix@^0.1.0:
version "0.1.0"
resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72"
+url-join@^4.0.0:
+ version "4.0.0"
+ resolved "https://registry.yarnpkg.com/url-join/-/url-join-4.0.0.tgz#4d3340e807d3773bda9991f8305acdcc2a665d2a"
+
url-parse-lax@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-1.0.0.tgz#7af8f303645e9bd79a272e7a14ac68bc0609da73"
dependencies:
prepend-http "^1.0.1"
+url-parse@1.4.3:
+ version "1.4.3"
+ resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.4.3.tgz#bfaee455c889023219d757e045fa6a684ec36c15"
+ dependencies:
+ querystringify "^2.0.0"
+ requires-port "^1.0.0"
+
url-regex@^3.0.0:
version "3.2.0"
resolved "https://registry.yarnpkg.com/url-regex/-/url-regex-3.2.0.tgz#dbad1e0c9e29e105dd0b1f09f6862f7fdb482724"
@@ -7535,6 +7719,10 @@ xml2js@0.4.19:
sax ">=0.6.0"
xmlbuilder "~9.0.1"
+xmlbuilder@^10.0.0:
+ version "10.1.1"
+ resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz#8cae6688cc9b38d850b7c8d3c0a4161dcaf475b0"
+
xmlbuilder@~9.0.1:
version "9.0.7"
resolved "http://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"