Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
# Montana Code School: Pet Project

# This is an important page for your demo day! Add your contributor names and
# a little about the project technologies being used - Harold

## Run
~~~
$ npm install
Expand Down
19 changes: 7 additions & 12 deletions helpers/scrape.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
// seems to work using import. good to resolve any remaining issues.
// Please only include the needed helper code now - not sure what is still being used
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you mean delete the Runner files that I used to test before we moved the functions to srcServer?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can keep the runner files if they help you test - but maybe rename them to testScrapeRunner.js and testSyncRunner.js.

// -Harold

let request = require('request');
// import request from 'request';
let cheerio = require('cheerio');
// import cheerio from 'cheerio';
let scraper = {};
import request from 'request';
import cheerio from 'cheerio';

let scraper = {};

scraper.scrapePetango = function(url, callback) {
//Make a GET request
Expand Down Expand Up @@ -71,10 +72,4 @@ scraper.parseIndividualAnimalResponse = function(html) {
return petObject;
};







module.exports = scraper;
export default scraper;
7 changes: 6 additions & 1 deletion helpers/scrape.test.js
Original file line number Diff line number Diff line change
@@ -1,14 +1,19 @@
import {expect} from 'chai';
let scrape = require('./scrape');
import scrape from './scrape';

describe('Scrape', () => {
// Still not reading from a file.
// You can do some expects around the number of animals in the data etc,
// No need to do a full .to.eql comparison. - Harold
//We need to figure out how to run callbacks and mock the data so that we can
//run this test.
xdescribe('scrapePetango', () => {
it('works', () => {
expect(scrape.scrapePetango("http://ws.petango.com/Webservices/adoptablesearch/wsAdoptableAnimals.aspx?species=Dog&sex=All&agegroup=All&colnum=1&authkey=1t4v495156y98t2wd78317102f933h83or1340ptjm31spd04d")).to.eql([]);
});
});
// rather than put this html string right in your code - would be better to
// save the html in a file and read it (using NodeJS file reading) - Harold
let html = `
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
Expand Down
1 change: 1 addition & 0 deletions helpers/scrapeAndSyncRunner.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
// since you are using babel-node on the server, you can use import - Harold
let mongoose = require("mongoose");
let uriUtil = require('mongodb-uri');
mongoose.Promise = global.Promise;
Expand Down
2 changes: 2 additions & 0 deletions helpers/sync.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
//You scrape petango and you get one pet object back.
//You want to look that pet up in your database by animalId.

// Let's get import working (and ES6 export at bottom)- Harold
let Pet = require('../models/pet');
let sync = {};

Expand Down
1 change: 1 addition & 0 deletions helpers/syncRunner.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
//Loki is not in the scrape and needs to be removed from the database.
//Baltazaar is in both the scrape and the DB so he should stay put.

// since you are using babel-node on the server, you can use import - Harold
let mongoose = require("mongoose");
let uriUtil = require('mongodb-uri');
mongoose.Promise = global.Promise;
Expand Down
1 change: 1 addition & 0 deletions models/pet.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
// since you are using babel-node on the server, you can use import - Harold
let mongoose = require("mongoose");

let PetSchema = new mongoose.Schema({
Expand Down
1 change: 1 addition & 0 deletions routes/petRoutes.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import React from 'react';
import webpack from 'webpack';
// Use import instead of require below - Harold
const Pet = require ('../models/pet');
let express = require('express');
let router = express.Router();
Expand Down
2 changes: 1 addition & 1 deletion routes/userRoutes.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import hash from 'password-hash';
import User from '../models/user';
import jwt from 'jsonwebtoken';
import configAuth from '../tools/configAuth';

// Use import below - Harold
let express = require('express');
let userRoutes = express.Router();
let app = express();
Expand Down
1 change: 1 addition & 0 deletions src/components/DisplayPets.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ export default class DisplayPets extends React.Component {
super();
this.state = {
petPics: [],
// Not clear why the window.location below? - Harold
species: window.location.hash.split("species=")[1].split("&")[0]
};
this.loadPetsFromDb = this.loadPetsFromDb.bind(this);
Expand Down
1 change: 1 addition & 0 deletions tools/configAuth.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
// This isn't safe - let's move this to an environment variable
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure what you mean? Store the secret in a variable?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup - instead of putting your secret on a public github file (configAuth.js), you could put it in an environment variable the way we manage PORT and MONGODB_URI in the srcServer.js file.

export default {
secret: 'ilovedogs'
};
9 changes: 5 additions & 4 deletions tools/srcServer.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import User from '../models/user';
import Pet from'../models/pet';
import petRoutes from '../routes/petRoutes';
import userRoutes from '../routes/userRoutes';
import scrapeRunner from '../helpers/scrape';
import syncRunner from '../helpers/sync';
/* eslint-disable no-console */

const port = process.env.PORT || 3000;
Expand Down Expand Up @@ -41,14 +43,13 @@ mongoose.connect(mongooseUri, options);
const PROD = process.env.NODE_ENV === 'production';

//Timed scrape and sync
let scrapeRunner = require('../helpers/scrape');
let syncRunner = require('../helpers/sync');
//

// Not too worried - but may look better to hide in an environment variable
let url = "http://ws.petango.com/Webservices/adoptablesearch/" +
"wsAdoptableAnimals.aspx?sex=All&agegroup=All&colnum=" +
"1&authkey=1t4v495156y98t2wd78317102f933h83or1340ptjm31spd04d";
//Call it when you npm start
// scrapeAndSync();
scrapeAndSync();
//Call again every hour
setInterval(scrapeAndSync, 3600000);

Expand Down