Switch to PostgreSQL

README.md

@@ -4,7 +4,7 @@ If you want to contribute join us on the [Pebble Dev Discord server](http://disc
 
 ## Requirements
 
-Backend/API layer requires `git`, `go`, `npm`, and `apib2swagger`.
+Backend/API layer requires `git`, `go`, `npm`, `postgresql` and `apib2swagger`.
 
 To make the backend do anything, you also need to download a copy of the Pebble App Store. You can already start downloading it [here](https://drive.google.com/file/d/0B1rumprSXUAhTjB1aU9GUFVPUW8/view) while you setup the development environment.
 
@@ -30,7 +30,8 @@ Instructions to setup the database:
 
 1. If you haven't already, download a copy of the Pebble App Store by using [this tool](https://github.com/azertyfun/PebbleAppStoreCrawler). To ease the load on fitbit's servers, you can download it directly [here](https://drive.google.com/file/d/0B1rumprSXUAhTjB1aU9GUFVPUW8/view);
 2. Extract the PebbleAppStore folder to the project directory: `tar -xzf PebbleAppStore.tar.gz -C $GOPATH/src/pebble-dev/rebblestore-api`;
-3. Start `./rebblestore-api` and access http://localhost:8080/admin/rebuild/db to rebuild the database.
+3. Setup the PostgreSQL database with a `rebblestore` user and a `rebblestore` database (as described on the [Arch Wiki](https://wiki.archlinux.org/index.php/PostgreSQL)), then [enable SSL](https://www.postgresql.org/docs/current/static/ssl-tcp.html) in `postgresql.conf`;
+4. Start `./rebblestore-api` and access http://localhost:8080/admin/rebuild/db to rebuild the database.
 
 ## Contributing
 

db/models.go

@@ -1,5 +1,7 @@
 package db
 
+import "time"
+
 // RebbleCard contains succint information about an app, to display on a result page for example
 type RebbleCard struct {
 	Id       string `json:"id"`
@@ -23,7 +25,7 @@ type RebbleApplication struct {
 	ThumbsUp           int           `json:"thumbs_up"`
 	Type               string        `json:"type"`
 	SupportedPlatforms []string      `json:"supported_platforms"`
-	Published          JSONTime      `json:"published_date"`
+	Published          time.Time     `json:"published_date"`
 	AppInfo            RebbleAppInfo `json:"appInfo"`
 	Assets             RebbleAssets  `json:"assets"`
 	DoomsdayBackup     bool          `json:"doomsday_backup"`
@@ -34,7 +36,7 @@ type RebbleAppInfo struct {
 	PbwUrl      string             `json:"pbwUrl"`
 	RebbleReady bool               `json:"rebbleReady"`
 	Tags        []RebbleCollection `json:"tags"`
-	Updated     JSONTime           `json:"updated"`
+	Updated     time.Time          `json:"updated"`
 	Version     string             `json:"version"`
 	SupportUrl  string             `json:"supportUrl"`
 	AuthorUrl   string             `json:"authorUrl"`
@@ -69,7 +71,7 @@ type RebbleScreenshotsPlatform struct {
 
 // RebbleVersion contains information about a specific version of an app
 type RebbleVersion struct {
-	Number      string   `json:"number"`
-	ReleaseDate JSONTime `json:"release_date"`
-	Description string   `json:"description"`
+	Number      string    `json:"number"`
+	ReleaseDate time.Time `json:"release_date"`
+	Description string    `json:"description"`
 }

db/queries.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"encoding/json"
 	"errors"
+	"log"
 	"regexp"
 	"strings"
 	"time"
@@ -24,10 +25,11 @@ func (handler Handler) Search(query string) (RebbleCards, error) {
 
 	var cards RebbleCards
 	rows, err := handler.Query(
-		"SELECT id, name, type, thumbs_up, screenshots FROM apps WHERE name LIKE ? ESCAPE '!' ORDER BY thumbs_up DESC LIMIT 12",
+		"SELECT id, name, type, thumbs_up, screenshots FROM apps WHERE name LIKE $1 ESCAPE '!' ORDER BY thumbs_up DESC LIMIT 12",
 		query,
 	)
 	if err != nil {
+		log.Printf("SQL error while searching for `%v`", query)
 		return cards, err
 	}
 	cards.Cards = make([]RebbleCard, 0)
@@ -37,10 +39,12 @@ func (handler Handler) Search(query string) (RebbleCards, error) {
 		var screenshots []RebbleScreenshotsPlatform
 		err = rows.Scan(&card.Id, &card.Title, &card.Type, &card.ThumbsUp, &screenshots_b)
 		if err != nil {
+			log.Printf("SQL error: Could not scan search results: %v", err)
 			return RebbleCards{}, err
 		}
 		err = json.Unmarshal(screenshots_b, &screenshots)
 		if err != nil {
+			log.Printf("Error: Could not unmarshal `screenshots`: %v", err)
 			return RebbleCards{}, err
 		}
 		if len(screenshots) != 0 && len(screenshots[0].Screenshots) != 0 {
@@ -61,11 +65,12 @@ func (handler Handler) GetAppsForCollection(collectionID string, sortByPopular b
 		order = "published_date"
 	}
 
-	row := handler.QueryRow("SELECT apps FROM collections WHERE id=?", collectionID)
+	row := handler.QueryRow("SELECT apps FROM collections WHERE id=$1", collectionID)
 	var appIdsB []byte
 	var appIds []string
 	err := row.Scan(&appIdsB)
 	if err != nil {
+		log.Printf("SQL error: Could not scan collection: %v", err)
 		return nil, err
 	}
 	json.Unmarshal(appIdsB, &appIds)
@@ -79,6 +84,7 @@ func (handler Handler) GetAppsForCollection(collectionID string, sortByPopular b
 	// There is no feasible way for idList to contain user generated data, and therefore for it to be a SQL injection vector. But just in case, we strip any non-authorized character
 	reg, err := regexp.Compile("[^a-zA-Z0-9, ']+")
 	if err != nil {
+		log.Printf("Error: Could not Compile regex: %v", err)
 		return nil, err
 	}
 	idList = reg.ReplaceAllString(idList, "")
@@ -88,6 +94,7 @@ func (handler Handler) GetAppsForCollection(collectionID string, sortByPopular b
 	// It is not possible to give a list or an order via a prepared statement, so this will have to do. We just sanitized idList, so SQL injection isn't a concern.
 	rows, err := handler.Query("SELECT id, name, type, thumbs_up, screenshots, published_date, supported_platforms FROM apps WHERE id IN (" + idList + ") ORDER BY " + order + " DESC")
 	if err != nil {
+		log.Printf("SQL error: Could not query app from collection: %v", err)
 		return nil, err
 	}
 
@@ -100,15 +107,18 @@ func (handler Handler) GetAppsForCollection(collectionID string, sortByPopular b
 		var screenshots_b []byte
 		err = rows.Scan(&app.Id, &app.Name, &app.Type, &app.ThumbsUp, &screenshots_b, &t, &supported_platforms_b)
 		if err != nil {
+			log.Printf("SQL error: Could not scan app for collection: %v", err)
 			return []RebbleApplication{}, err
 		}
-		app.Published.Time = time.Unix(0, t)
+		app.Published = time.Unix(0, t)
 		err = json.Unmarshal(supported_platforms_b, &app.SupportedPlatforms)
 		if err != nil {
+			log.Printf("Error: Could not unmarshal `supported_platforms`: %v", err)
 			return []RebbleApplication{}, err
 		}
 		err = json.Unmarshal(screenshots_b, &app.Assets.Screenshots)
 		if err != nil {
+			log.Printf("Error: Could not unmarshal `screenshots`: %v", err)
 			return []RebbleApplication{}, err
 		}
 		apps = append(apps, app)
@@ -118,8 +128,9 @@ func (handler Handler) GetAppsForCollection(collectionID string, sortByPopular b
 
 // GetCollectionName returns the name of a collection
 func (handler Handler) GetCollectionName(collectionID string) (string, error) {
-	rows, err := handler.Query("SELECT name FROM collections WHERE id=?", collectionID)
+	rows, err := handler.Query("SELECT name FROM collections WHERE id=$1", collectionID)
 	if err != nil {
+		log.Printf("SQL error: Could not SELECT collection name: %v", err)
 		return "", err
 	}
 	if !rows.Next() {
@@ -128,6 +139,7 @@ func (handler Handler) GetCollectionName(collectionID string) (string, error) {
 	var name string
 	err = rows.Scan(&name)
 	if err != nil {
+		log.Printf("SQL error: Could not scan collection name: %v", err)
 		return "", err
 	}
 
@@ -158,18 +170,17 @@ func (handler Handler) GetAllApps(sortby string, ascending bool, offset int, lim
 		FROM apps
 		JOIN authors ON apps.author_id = authors.id
 		ORDER BY `+orderCol+" "+order+`
-		LIMIT ?
-		OFFSET ?
+		LIMIT $1
+		OFFSET $1
 	`, limit, offset)
 	if err != nil {
+		log.Printf("SQL error: Could not select all apps: %v", err)
 		return nil, err
 	}
 	apps := make([]RebbleApplication, 0)
 	for rows.Next() {
 		app := RebbleApplication{}
-		var t_published int64
-		err = rows.Scan(&app.Name, &app.Author.Name, &app.Assets.Icon, &app.Id, &app.ThumbsUp, &t_published)
-		app.Published.Time = time.Unix(0, t_published)
+		err = rows.Scan(&app.Name, &app.Author.Name, &app.Assets.Icon, &app.Id, &app.ThumbsUp, &app.Published)
 
 		apps = append(apps, app)
 	}
@@ -178,35 +189,38 @@ func (handler Handler) GetAllApps(sortby string, ascending bool, offset int, lim
 
 // GetApp returns a specific app
 func (handler Handler) GetApp(id string) (RebbleApplication, error) {
-	row := handler.QueryRow("SELECT apps.id, apps.name, apps.author_id, authors.name, apps.tag_ids, apps.description, apps.thumbs_up, apps.type, apps.supported_platforms, apps.published_date, apps.pbw_url, apps.rebble_ready, apps.updated, apps.version, apps.support_url, apps.author_url, apps.source_url, apps.screenshots, apps.banner_url, apps.icon_url, apps.doomsday_backup FROM apps JOIN authors ON apps.author_id = authors.id WHERE apps.id=?", id)
+	row := handler.QueryRow(`
+		SELECT apps.id, apps.name, apps.author_id, authors.name, apps.tag_ids, apps.description, apps.thumbs_up, apps.type, apps.supported_platforms, apps.published_date, apps.pbw_url, apps.rebble_ready, apps.updated, apps.version, apps.support_url, apps.author_url, apps.source_url, apps.screenshots, apps.banner_url, apps.icon_url, apps.doomsday_backup FROM apps
+			JOIN authors ON apps.author_id = authors.id
+			WHERE apps.id=$1
+	`, id)
 
 	app := RebbleApplication{}
 	var supportedPlatforms_b []byte
-	var t_published, t_updated int64
 	var tagIds_b []byte
 	var tagIds []string
 	var screenshots_b []byte
 	var screenshots *([]RebbleScreenshotsPlatform)
-	err := row.Scan(&app.Id, &app.Name, &app.Author.Id, &app.Author.Name, &tagIds_b, &app.Description, &app.ThumbsUp, &app.Type, &supportedPlatforms_b, &t_published, &app.AppInfo.PbwUrl, &app.AppInfo.RebbleReady, &t_updated, &app.AppInfo.Version, &app.AppInfo.SupportUrl, &app.AppInfo.AuthorUrl, &app.AppInfo.SourceUrl, &screenshots_b, &app.Assets.Banner, &app.Assets.Icon, &app.DoomsdayBackup)
+	err := row.Scan(&app.Id, &app.Name, &app.Author.Id, &app.Author.Name, &tagIds_b, &app.Description, &app.ThumbsUp, &app.Type, &supportedPlatforms_b, &app.Published, &app.AppInfo.PbwUrl, &app.AppInfo.RebbleReady, &app.AppInfo.Updated, &app.AppInfo.Version, &app.AppInfo.SupportUrl, &app.AppInfo.AuthorUrl, &app.AppInfo.SourceUrl, &screenshots_b, &app.Assets.Banner, &app.Assets.Icon, &app.DoomsdayBackup)
 	if err == sql.ErrNoRows {
 		return RebbleApplication{}, errors.New("No application with this ID")
 	} else if err != nil {
+		log.Printf("SQL error: Could not SELECT app: %v", err)
 		return RebbleApplication{}, err
 	}
 
 	json.Unmarshal(supportedPlatforms_b, &app.SupportedPlatforms)
-	app.Published.Time = time.Unix(0, t_published)
-	app.AppInfo.Updated.Time = time.Unix(0, t_updated)
 	json.Unmarshal(tagIds_b, &tagIds)
 	app.AppInfo.Tags = make([]RebbleCollection, len(tagIds))
 	json.Unmarshal(screenshots_b, &screenshots)
 	app.Assets.Screenshots = screenshots
 
 	for i, tagID := range tagIds {
-		row := handler.QueryRow("SELECT id, name, color FROM collections WHERE id=?", tagID)
+		row := handler.QueryRow("SELECT id, name, color FROM collections WHERE id=$1", tagID)
 
 		err = row.Scan(&app.AppInfo.Tags[i].Id, &app.AppInfo.Tags[i].Name, &app.AppInfo.Tags[i].Color)
 		if err != nil {
+			log.Printf("SQL error: Could not scan app: %v", err)
 			return RebbleApplication{}, err
 		}
 	}
@@ -216,33 +230,38 @@ func (handler Handler) GetApp(id string) (RebbleApplication, error) {
 
 // GetAppTags returns the the list of tags of the application with the id `id`
 func (handler Handler) GetAppTags(id string) ([]RebbleCollection, error) {
-	rows, err := handler.Query("SELECT apps.tag_ids FROM apps WHERE id=?", id)
+	rows, err := handler.Query("SELECT apps.tag_ids FROM apps WHERE id=$1", id)
 	if err != nil {
+		log.Printf("SQL error: Could not SELECT app tags: %v", err)
 		return []RebbleCollection{}, err
 	}
 	exists := rows.Next()
 	if !exists {
+		log.Printf("Error: Tag `%v` doesn't exist!", id)
 		return []RebbleCollection{}, err
 	}
 
 	var tagIds_b []byte
 	var tagIds []string
 	err = rows.Scan(&tagIds_b)
 	if err != nil {
+		log.Printf("SQL error: Could not scan app tags: %v", err)
 		return []RebbleCollection{}, err
 	}
 	json.Unmarshal(tagIds_b, &tagIds)
 	collections := make([]RebbleCollection, len(tagIds))
 
 	for i, tagId := range tagIds {
-		rows, err := handler.Query("SELECT id, name, color FROM collections WHERE id=?", tagId)
+		rows, err := handler.Query("SELECT id, name, color FROM collections WHERE id=$1", tagId)
 		if err != nil {
+			log.Printf("SQL error: Could not SELECT collection from app tag: %v", err)
 			return []RebbleCollection{}, err
 		}
 
 		rows.Next()
 		err = rows.Scan(&collections[i].Id, &collections[i].Name, &collections[i].Color)
 		if err != nil {
+			log.Printf("SQL error: Could not scan collection from app tag: %v", err)
 			return []RebbleCollection{}, err
 		}
 	}
@@ -252,8 +271,9 @@ func (handler Handler) GetAppTags(id string) ([]RebbleCollection, error) {
 
 // GetAppVersions returns the the list of versions of the application with the id `id`
 func (handler Handler) GetAppVersions(id string) ([]RebbleVersion, error) {
-	rows, err := handler.Query("SELECT apps.versions FROM apps WHERE id=?", id)
+	rows, err := handler.Query("SELECT apps.versions FROM apps WHERE id=$1", id)
 	if err != nil {
+		log.Printf("SQL error: Could not SELECT app versions: %v", err)
 		return []RebbleVersion{}, err
 	}
 	exists := rows.Next()
@@ -265,6 +285,7 @@ func (handler Handler) GetAppVersions(id string) ([]RebbleVersion, error) {
 	var versions []RebbleVersion
 	err = rows.Scan(&versions_b)
 	if err != nil {
+		log.Printf("SQL error: Could not scan app versions: %v", err)
 		return []RebbleVersion{}, err
 	}
 	json.Unmarshal(versions_b, &versions)
@@ -274,8 +295,9 @@ func (handler Handler) GetAppVersions(id string) ([]RebbleVersion, error) {
 
 // GetAuthor returns a RebbleAuthor
 func (handler Handler) GetAuthor(id int) (RebbleAuthor, error) {
-	rows, err := handler.Query("SELECT authors.name FROM authors WHERE id=?", id)
+	rows, err := handler.Query("SELECT authors.name FROM authors WHERE id=$1", id)
 	if err != nil {
+		log.Printf("SQL error: Could not SELECT author: %v", err)
 		return RebbleAuthor{}, err
 	}
 	exists := rows.Next()
@@ -288,6 +310,7 @@ func (handler Handler) GetAuthor(id int) (RebbleAuthor, error) {
 	}
 	err = rows.Scan(&author.Name)
 	if err != nil {
+		log.Printf("SQL error: Could not scan author: %v", err)
 		return RebbleAuthor{}, err
 	}
 
@@ -299,10 +322,11 @@ func (handler Handler) GetAuthorCards(id int) (RebbleCards, error) {
 	rows, err := handler.Query(`
 		SELECT id, name, type, screenshots, thumbs_up
 		FROM apps
-		WHERE author_id=?
+		WHERE author_id=$1
 		ORDER BY published_date ASC
 	`, id)
 	if err != nil {
+		log.Printf("SQL error: Could not SELECT author cards: %v", err)
 		return RebbleCards{}, err
 	}
 
@@ -318,11 +342,13 @@ func (handler Handler) GetAuthorCards(id int) (RebbleCards, error) {
 
 		err = rows.Scan(&card.Id, &card.Title, &card.Type, &screenshots_b, &card.ThumbsUp)
 		if err != nil {
+			log.Printf("SQL error: Could not scan author card: %v", err)
 			return RebbleCards{}, err
 		}
 
 		err = json.Unmarshal(screenshots_b, &screenshots)
 		if err != nil {
+			log.Printf("Error: could not unmarshal author card `screenshots`", err)
 			return RebbleCards{}, err
 		}
 		if len(screenshots) != 0 && len(screenshots[0].Screenshots) != 0 {

db/types.go

@@ -1,30 +1 @@
 package db
-
-import (
-	"time"
-)
-
-// JSONTime is a dummy time object that is meant to allow Go's JSON module to
-// properly de-serialize the JSON time format.
-type JSONTime struct {
-	time.Time
-}
-
-// UnmarshalJSON allows for the custom time format within the application JSON
-// to be decoded into Go's native time format.
-func (self *JSONTime) UnmarshalJSON(b []byte) (err error) {
-	s := string(b)
-
-	// Return an empty time.Time object if it didn't exist in the first place.
-	if s == "null" {
-		self.Time = time.Time{}
-		return
-	}
-
-	t, err := time.Parse("\"2006-01-02T15:04:05.999Z\"", s)
-	if err != nil {
-		t = time.Time{}
-	}
-	self.Time = t
-	return
-}

main.go

@@ -11,7 +11,7 @@ import (
 	"pebble-dev/rebblestore-api/rebbleHandlers"
 
 	"github.com/gorilla/handlers"
-	_ "github.com/mattn/go-sqlite3"
+	_ "github.com/lib/pq"
 	"github.com/pborman/getopt"
 )
 
@@ -27,7 +27,7 @@ func main() {
 		return
 	}
 
-	database, err := sql.Open("sqlite3", "./RebbleAppStore.db")
+	database, err := sql.Open("postgres", "user=rebblestore dbname=rebblestore")
 	if err != nil {
 		panic("Could not connect to database" + err.Error())
 	}

main_test.go

@@ -11,15 +11,15 @@ import (
 	"github.com/adams-sarah/test2doc/test"
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
-	_ "github.com/mattn/go-sqlite3"
+	_ "github.com/lib/pq"
 )
 
 var server *test.Server
 
 func TestMain(m *testing.M) {
 	var err error
 
-	database, err := sql.Open("sqlite3", "./RebbleAppStore.db")
+	database, err := sql.Open("postgres", "user=rebblestore dbname=rebblestore")
 	if err != nil {
 		panic("Could not connect to database" + err.Error())
 	}