diff --git a/README.md b/README.md index 9c8a375..e673913 100644 --- a/README.md +++ b/README.md @@ -53,10 +53,11 @@ docker run -v $PWD:/host --entrypoint cp cndy-store/analytics analytics /host/cn ## Latest stats -GET https://api.cndy.store/stats/latest +GET https://api.cndy.store/stats/latest?asset_code=CNDY&asset_issuer=GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX ```json { + "status": "ok", "latest": { "paging_token": "33825130903777281-1", "asset_type": "credit_alphanum4", @@ -74,12 +75,13 @@ GET https://api.cndy.store/stats/latest ## Asset stats history -GET https://api.cndy.store/stats[?from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z] +GET https://api.cndy.store/stats?asset_code=CNDY&asset_issuer=GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX[&from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z] If not set, `from` defaults to UNIX timestamp `0`, `to` to `now`. ```json { + "status": "ok", "stats": [ { "paging_token": "33864305300480001-1", @@ -115,18 +117,21 @@ GET https://api.cndy.store/stats/cursor ```json { + "status": "ok", "current_cursor": "33877250331906049-1" } ``` ## Effects -GET https://api.cndy.store/effects[?from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z] +GET https://api.cndy.store/effects?asset_code=CNDY&asset_issuer=GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX[&from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z] + If not set, `from` defaults to UNIX timestamp `0`, `to` to `now`. ```json { + "status": "ok", "effects": [ { "id": "0033819672000335873-0000000001", @@ -174,3 +179,56 @@ If not set, `from` defaults to UNIX timestamp `0`, `to` to `now`. } } ``` + + +## Assets + +### Create a new asset + +POST https://api.cndy.store/assets + +Body + +```json +{ + "code": "CNDY", + "issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX" +} +``` + +Response + +```json +{ + "status": "ok", + "asset": { + "code": "CNDY", + "issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX", + "created_at": "2018-07-04T19:16:47.02965Z" + } +} +``` + +### Get all known assets + +GET https://api.cndy.store/assets + + +```json +{ + "status": "ok", + "assets": [ + { + "type": "credit_alphanum4", + "code": "CNDY", + "issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX", + "created_at": "2018-07-04T19:16:47.02965Z" + }, + { + "code": "LOCALCOIN", + "issuer": "GCJKCXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", + "created_at": "2018-07-04T19:54:39.14328Z" + } + ] +} +``` diff --git a/controllers/assets/assets.go b/controllers/assets/assets.go new file mode 100644 index 0000000..5f156ca --- /dev/null +++ b/controllers/assets/assets.go @@ -0,0 +1,76 @@ +package assets + +import ( + "fmt" + "github.com/cndy-store/analytics/models/asset" + "github.com/cndy-store/analytics/utils/sql" + "github.com/gin-gonic/gin" + "log" + "net/http" +) + +func Init(db sql.Database, router *gin.Engine) { + router.POST("/assets", func(c *gin.Context) { + // Read JSON body and parse it into asset struct + body := asset.Asset{} + err := c.BindJSON(&body) + if err != nil { + jsonErrorMsg := fmt.Sprintf("Invalid JSON body: %s", err) + c.JSON(http.StatusBadRequest, gin.H{ + "status": "error", + "message": jsonErrorMsg, + }) + return + } + + exists, err := asset.Exists(db, body) + if err != nil { + log.Printf("[ERROR] POST /assets: %s", err) + c.JSON(http.StatusInternalServerError, gin.H{ + "status": "error", + "message": "Internal server error", + }) + return + } + if exists { + c.JSON(http.StatusConflict, gin.H{ + "status": "error", + "message": "Asset already exists", + }) + return + } + + newAsset, err := asset.New(db, body) + if err != nil { + log.Printf("[ERROR] POST /assets: %s", err) + c.JSON(http.StatusInternalServerError, gin.H{ + "status": "error", + "message": "Internal server error", + }) + return + } + + c.JSON(http.StatusOK, gin.H{ + "status": "ok", + "asset": newAsset, + }) + }) + + router.GET("/assets", func(c *gin.Context) { + assets, err := asset.Get(db) + if err != nil { + log.Printf("[ERROR] Couldn't get assets from database: %s", err) + c.JSON(http.StatusInternalServerError, gin.H{ + "status": "error", + "message": "Internal server error", + }) + return + } + + c.JSON(http.StatusOK, gin.H{ + "status": "ok", + "assets": assets, + }) + return + }) +} diff --git a/controllers/assets/assets_test.go b/controllers/assets/assets_test.go new file mode 100644 index 0000000..0682f30 --- /dev/null +++ b/controllers/assets/assets_test.go @@ -0,0 +1,136 @@ +package assets + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/cndy-store/analytics/utils/cndy" + "github.com/cndy-store/analytics/utils/sql" + "github.com/cndy-store/analytics/utils/test" + "github.com/gin-gonic/gin" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +type HttpTest struct { + method string + url string + body string + statusCode int + expectedBody []string +} + +func TestAssets(t *testing.T) { + db, err := sql.OpenAndMigrate("../..") + if err != nil { + t.Error(err) + } + + tx, err := db.Beginx() + if err != nil { + t.Error(err) + } + defer tx.Rollback() + + err = test.InsertTestData(tx) + if err != nil { + t.Error(err) + } + + var tests = []HttpTest{ + { + "POST", + "/assets", + `{"code": "TEST", "issuer": "GCJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}`, + http.StatusOK, + []string{ + `"status":"ok"`, + `"code":"TEST"`, + `"issuer":"GCJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"`, + }, + }, + + // Check whether duplicates are prevented + { + "POST", + "/assets", + `{"code": "TEST", "issuer": "GCJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}`, + http.StatusConflict, + []string{ + `"status":"error"`, + `"message":"Asset already exists"`, + }, + }, + + { + "POST", + "/assets", + `{"code": "invalid`, + http.StatusBadRequest, + []string{ + `"status":"error"`, + }, + }, + + // Check whether new asset as well as CNDY asset are present in database + { + "GET", + "/assets", + "", + http.StatusOK, + []string{ + `"status":"ok"`, + fmt.Sprintf(`"code":"%s"`, cndy.AssetCode), + fmt.Sprintf(`"issuer":"%s"`, cndy.AssetIssuer), + `"code":"TEST"`, + `"issuer":"GCJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"`, + }, + }, + } + + router := gin.Default() + Init(tx, router) + + for _, tt := range tests { + body := bytes.NewBufferString(tt.body) + req, _ := http.NewRequest(tt.method, tt.url, body) + resp := httptest.NewRecorder() + + router.ServeHTTP(resp, req) + + if resp.Code != tt.statusCode { + t.Errorf("Expected code %v, got %v, for %+v", tt.statusCode, resp.Code, tt) + } + + type resJson struct { + Status string + } + + if tt.statusCode == http.StatusOK { + if !strings.Contains(resp.Body.String(), `"status":"ok"`) { + t.Errorf("Body did not contain ok status message: %s", resp.Body.String()) + } + } else { + if !strings.Contains(resp.Body.String(), `"status":"error"`) { + t.Errorf("Body did not contain error status message: %s", resp.Body.String()) + } + + // Skip to next test + continue + } + + res := resJson{} + err := json.Unmarshal([]byte(resp.Body.String()), &res) + if err != nil { + t.Error(err) + } + + for _, contains := range tt.expectedBody { + if !strings.Contains(resp.Body.String(), contains) { + t.Errorf("Body did not contain '%s' in '%s'", contains, resp.Body.String()) + } + } + } +} diff --git a/controllers/docs/docs.go b/controllers/docs/docs.go index f02fdf8..84ab695 100644 --- a/controllers/docs/docs.go +++ b/controllers/docs/docs.go @@ -16,107 +16,146 @@ func Init(router *gin.Engine) { // This data was generated from the corresponding README.md section using pandoc: // pandoc README.md -o docs.html const htmlData = ` -

API documentation

+

API endpoints and examples

Latest stats

-

GET https://api.cndy.store/stats/latest

+

GET https://api.cndy.store/stats/latest?asset_code=CNDY&asset_issuer=GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX

{
-	  "latest": {
-	    "paging_token": "33825130903777281-1",
-	    "asset_type": "credit_alphanum4",
-	    "asset_code": "CNDY",
-	    "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX",
-	    "payments": 4,
-	    "accounts_with_trustline": 4,
-	    "accounts_with_payments": 2,
-	    "created_at": "2018-03-12T18:49:40Z",
-	    "issued": "2000.0000000",
-	    "transferred": "40.0000000"
-	  }
-	}
+ "status": "ok", + "latest": { + "paging_token": "33825130903777281-1", + "asset_type": "credit_alphanum4", + "asset_code": "CNDY", + "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX", + "payments": 4, + "accounts_with_trustline": 4, + "accounts_with_payments": 2, + "created_at": "2018-03-12T18:49:40Z", + "issued": "2000.0000000", + "transferred": "40.0000000" + } + }

Asset stats history

-

GET https://api.cndy.store/stats[?from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z]

+

GET https://api.cndy.store/stats?asset_code=CNDY&asset_issuer=GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX[&from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z]

If not set, from defaults to UNIX timestamp 0, to to now.

{
-	  "stats": [
-	    {
-	      "paging_token": "33864305300480001-1",
-	      "asset_type": "credit_alphanum4",
-	      "asset_code": "CNDY",
-	      "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX",
-	      "payments": 6,
-	      "accounts_with_trustline": 5,
-	      "accounts_with_payments": 2,
-	      "created_at": "2018-03-13T07:29:48Z",
-	      "issued": "2000.0000000",
-	      "transferred": "140.0000000"
-	    },
-	    {
-	      "paging_token": "33864305300480001-2",
-	      "asset_type": "credit_alphanum4",
-	      "asset_code": "CNDY",
-	      "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX",
-	      "payments": 6,
-	      "accounts_with_trustline": 5,
-	      "accounts_with_payments": 2,
-	      "created_at": "2018-03-13T07:29:48Z",
-	      "issued": "3000.0000000",
-	      "transferred": "140.0000000"
-	    }
-	  ]
-	}
+ "status": "ok", + "stats": [ + { + "paging_token": "33864305300480001-1", + "asset_type": "credit_alphanum4", + "asset_code": "CNDY", + "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX", + "payments": 6, + "accounts_with_trustline": 5, + "accounts_with_payments": 2, + "created_at": "2018-03-13T07:29:48Z", + "issued": "2000.0000000", + "transferred": "140.0000000" + }, + { + "paging_token": "33864305300480001-2", + "asset_type": "credit_alphanum4", + "asset_code": "CNDY", + "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX", + "payments": 6, + "accounts_with_trustline": 5, + "accounts_with_payments": 2, + "created_at": "2018-03-13T07:29:48Z", + "issued": "3000.0000000", + "transferred": "140.0000000" + } + ] + }

Current Horizon cursor

GET https://api.cndy.store/stats/cursor

{
-	  "current_cursor": "33877250331906049-1"
-	}
+ "status": "ok", + "current_cursor": "33877250331906049-1" + }

Effects

-

GET https://api.cndy.store/effects[?from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z]

+

GET https://api.cndy.store/effects?asset_code=CNDY&asset_issuer=GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX[&from=2018-03-03T23:05:40Z&to=2018-03-03T23:05:50Z]

If not set, from defaults to UNIX timestamp 0, to to now.

{
-	  "effects": [
-	    {
-	      "id": "0033819672000335873-0000000001",
-	      "operation": "https://horizon-testnet.stellar.org/operations/33819672000335873",
-	      "succeeds": "https://horizon-testnet.stellar.org/effects?order=desc&cursor=33819672000335873-1",
-	      "precedes": "https://horizon-testnet.stellar.org/effects?order=asc&cursor=33819672000335873-1",
-	      "paging_token": "33819672000335873-1",
-	      "account": "GDNH64DRUT4CY3UJLWQIB655PQ6OG34UGYB4NC5DC4TYWLNJIBCEYTTD",
-	      "type": "trustline_created",
-	      "type_i": 20,
-	      "starting_balance": "",
-	      "asset_type": "credit_alphanum4",
-	      "asset_code": "CNDY",
-	      "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX",
-	      "signer_public_key": "",
-	      "signer_weight": 0,
-	      "signer_key": "",
-	      "signer_type": "",
-	      "created_at": "2018-03-12T17:03:45Z",
-	      "amount": "0.0000000",
-	      "balance": "0.0000000",
-	      "balance_limit": "922337203685.4775807"
-	    },
-	    {
-	      "id": "0033820110087000065-0000000002",
-	      "operation": "https://horizon-testnet.stellar.org/operations/33820110087000065",
-	      "succeeds": "https://horizon-testnet.stellar.org/effects?order=desc&cursor=33820110087000065-2",
-	      "precedes": "https://horizon-testnet.stellar.org/effects?order=asc&cursor=33820110087000065-2",
-	      "paging_token": "33820110087000065-2",
-	      "account": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX",
-	      "type": "account_debited",
-	      "type_i": 3,
-	      "starting_balance": "",
-	      "asset_type": "credit_alphanum4",
-	      "asset_code": "CNDY",
-	      "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX",
-	      "signer_public_key": "",
-	      "signer_weight": 0,
-	      "signer_key": "",
-	      "signer_type": "",
-	      "created_at": "2018-03-12T17:12:15Z",
-	      "amount": "1000.0000000",
-	      "balance": "0.0000000",
-	      "balance_limit": "0.0000000"
-	    }
-	}
+ "status": "ok", + "effects": [ + { + "id": "0033819672000335873-0000000001", + "operation": "https://horizon-testnet.stellar.org/operations/33819672000335873", + "succeeds": "https://horizon-testnet.stellar.org/effects?order=desc&cursor=33819672000335873-1", + "precedes": "https://horizon-testnet.stellar.org/effects?order=asc&cursor=33819672000335873-1", + "paging_token": "33819672000335873-1", + "account": "GDNH64DRUT4CY3UJLWQIB655PQ6OG34UGYB4NC5DC4TYWLNJIBCEYTTD", + "type": "trustline_created", + "type_i": 20, + "starting_balance": "", + "asset_type": "credit_alphanum4", + "asset_code": "CNDY", + "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX", + "signer_public_key": "", + "signer_weight": 0, + "signer_key": "", + "signer_type": "", + "created_at": "2018-03-12T17:03:45Z", + "amount": "0.0000000", + "balance": "0.0000000", + "balance_limit": "922337203685.4775807" + }, + { + "id": "0033820110087000065-0000000002", + "operation": "https://horizon-testnet.stellar.org/operations/33820110087000065", + "succeeds": "https://horizon-testnet.stellar.org/effects?order=desc&cursor=33820110087000065-2", + "precedes": "https://horizon-testnet.stellar.org/effects?order=asc&cursor=33820110087000065-2", + "paging_token": "33820110087000065-2", + "account": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX", + "type": "account_debited", + "type_i": 3, + "starting_balance": "", + "asset_type": "credit_alphanum4", + "asset_code": "CNDY", + "asset_issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX", + "signer_public_key": "", + "signer_weight": 0, + "signer_key": "", + "signer_type": "", + "created_at": "2018-03-12T17:12:15Z", + "amount": "1000.0000000", + "balance": "0.0000000", + "balance_limit": "0.0000000" + } + } +

Assets

+

Create a new asset

+

POST https://api.cndy.store/assets

+

Body

+ 
{
+	  "code": "CNDY",
+	  "issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX"
+	}
+

Response

+ 
{
+	  "status": "ok",
+	  "asset": {
+	    "code": "CNDY",
+	    "issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX",
+	    "created_at": "2018-07-04T19:16:47.02965Z"
+	  }
+	}
+

Get all known assets

+

GET https://api.cndy.store/assets

+ 
{
+	  "status": "ok",
+	  "assets": [
+	    {
+	      "type": "credit_alphanum4",
+	      "code": "CNDY",
+	      "issuer": "GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX",
+	      "created_at": "2018-07-04T19:16:47.02965Z"
+	    },
+	    {
+	      "code": "LOCALCOIN",
+	      "issuer": "GCJKCXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
+	      "created_at": "2018-07-04T19:54:39.14328Z"
+	    }
+	  ]
+	}
` diff --git a/controllers/effects/effects.go b/controllers/effects/effects.go index 310fdfc..41cc833 100644 --- a/controllers/effects/effects.go +++ b/controllers/effects/effects.go @@ -3,15 +3,16 @@ package effects import ( "github.com/cndy-store/analytics/models/effect" "github.com/cndy-store/analytics/utils/filter" + "github.com/cndy-store/analytics/utils/sql" "github.com/gin-gonic/gin" "log" "net/http" ) -func Init(db interface{}, router *gin.Engine) { +func Init(db sql.Database, router *gin.Engine) { // GET /effects[?from=XXX&to=XXX] router.GET("/effects", func(c *gin.Context) { - from, to, err := filter.Parse(c) + args, err := filter.Parse(c) if err != nil { log.Printf("[ERROR] Couldn't parse URL parameters: %s", err) c.JSON(http.StatusBadRequest, gin.H{ @@ -21,14 +22,18 @@ func Init(db interface{}, router *gin.Engine) { return } - effects, err := effect.Get(db, effect.Filter{From: from, To: to}) + effects, err := effect.Get(db, args) if err != nil { log.Printf("[ERROR] Couldn't get effect from database: %s", err) - c.String(http.StatusInternalServerError, "") + c.JSON(http.StatusInternalServerError, gin.H{ + "status": "error", + "message": "Internal server error", + }) return } c.JSON(http.StatusOK, gin.H{ + "status": "ok", "effects": effects, }) return diff --git a/controllers/effects/effects_test.go b/controllers/effects/effects_test.go index 16529e5..3f3eedb 100644 --- a/controllers/effects/effects_test.go +++ b/controllers/effects/effects_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/cndy-store/analytics/models/effect" + "github.com/cndy-store/analytics/utils/cndy" "github.com/cndy-store/analytics/utils/sql" "github.com/cndy-store/analytics/utils/test" "github.com/gin-gonic/gin" @@ -43,7 +44,7 @@ func TestEffects(t *testing.T) { var tests = []HttpTest{ { "GET", - "/effects", + fmt.Sprintf("/effects?asset_code=%s&asset_issuer=%s", cndy.AssetCode, cndy.AssetIssuer), "", http.StatusOK, test.Effects, @@ -52,7 +53,7 @@ func TestEffects(t *testing.T) { // Filter{From} { "GET", - fmt.Sprintf("/effects?from=%s", test.Effects[5].CreatedAt.Format(time.RFC3339)), + fmt.Sprintf("/effects?asset_code=%s&asset_issuer=%s&from=%s", cndy.AssetCode, cndy.AssetIssuer, test.Effects[5].CreatedAt.Format(time.RFC3339)), "", http.StatusOK, test.Effects[5:], @@ -61,7 +62,7 @@ func TestEffects(t *testing.T) { // Filter{To} { "GET", - fmt.Sprintf("/effects?to=%s", test.Effects[2].CreatedAt.Format(time.RFC3339)), + fmt.Sprintf("/effects?asset_code=%s&asset_issuer=%s&to=%s", cndy.AssetCode, cndy.AssetIssuer, test.Effects[2].CreatedAt.Format(time.RFC3339)), "", http.StatusOK, test.Effects[:3], @@ -70,43 +71,74 @@ func TestEffects(t *testing.T) { // Filter{From, To} { "GET", - fmt.Sprintf("/effects?from=%s&to=%s", test.Effects[3].CreatedAt.Format(time.RFC3339), test.Effects[4].CreatedAt.Format(time.RFC3339)), + fmt.Sprintf("/effects?asset_code=%s&asset_issuer=%s&from=%s&to=%s", cndy.AssetCode, cndy.AssetIssuer, test.Effects[3].CreatedAt.Format(time.RFC3339), test.Effects[4].CreatedAt.Format(time.RFC3339)), "", http.StatusOK, test.Effects[3:5], }, + + // Invalid Filter{} + { + "GET", + fmt.Sprintf("/effects?asset_code=%s&asset_issuer=%s&from=xxx", cndy.AssetCode, cndy.AssetIssuer), + "", + http.StatusBadRequest, + nil, + }, + + // Missing asset_code and asset_issuer + { + "GET", + "/effects", + "", + http.StatusBadRequest, + nil, + }, } router := gin.Default() Init(tx, router) - for _, test := range tests { - body := bytes.NewBufferString(test.body) - req, _ := http.NewRequest(test.method, test.url, body) + for _, tt := range tests { + body := bytes.NewBufferString(tt.body) + req, _ := http.NewRequest(tt.method, tt.url, body) resp := httptest.NewRecorder() router.ServeHTTP(resp, req) - if resp.Code != test.statusCode { - t.Errorf("Expected code %v, got %v, for %+v", test.statusCode, resp.Code, test) + if resp.Code != tt.statusCode { + t.Errorf("Expected code %v, got %v, for %+v", tt.statusCode, resp.Code, tt) } - effects := make(map[string][]effect.Effect) - err := json.Unmarshal([]byte(resp.Body.String()), &effects) - if err != nil { - t.Error(err) + type resJson struct { + Status string + Effects []effect.Effect + } + + if tt.statusCode == http.StatusOK { + if !strings.Contains(resp.Body.String(), `"status":"ok"`) { + t.Errorf("Body did not contain ok status message: %s", resp.Body.String()) + } + } else { + if !strings.Contains(resp.Body.String(), `"status":"error"`) { + t.Errorf("Body did not contain error status message: %s", resp.Body.String()) + } + + // Skip to next test + continue } - _, ok := effects["effects"] - if !ok { - t.Error(`Expected element "effects" in JSON response`) + res := resJson{} + err := json.Unmarshal([]byte(resp.Body.String()), &res) + if err != nil { + t.Error(err) } - if len(effects["effects"]) != len(test.expectedStats) { - t.Errorf("Expected %d JSON elements, got %d", len(test.expectedStats), len(effects["effects"])) + if len(res.Effects) != len(tt.expectedStats) { + t.Errorf("Expected %d JSON elements, got %d", len(tt.expectedStats), len(res.Effects)) } - for _, e := range test.expectedStats { + for _, e := range tt.expectedStats { var s []string s = append(s, fmt.Sprintf(`"paging_token":"%s"`, e.PagingToken)) s = append(s, fmt.Sprintf(`"account":"%s"`, e.Account)) diff --git a/controllers/stats/stats.go b/controllers/stats/stats.go index cdb977b..49b67a5 100644 --- a/controllers/stats/stats.go +++ b/controllers/stats/stats.go @@ -4,15 +4,16 @@ import ( "github.com/cndy-store/analytics/models/asset_stat" "github.com/cndy-store/analytics/models/cursor" "github.com/cndy-store/analytics/utils/filter" + "github.com/cndy-store/analytics/utils/sql" "github.com/gin-gonic/gin" "log" "net/http" ) -func Init(db interface{}, router *gin.Engine) { +func Init(db sql.Database, router *gin.Engine) { // GET /stats[?from=XXX&to=XXX] router.GET("/stats", func(c *gin.Context) { - from, to, err := filter.Parse(c) + args, err := filter.Parse(c) if err != nil { log.Printf("[ERROR] Couldn't parse URL parameters: %s", err) c.JSON(http.StatusBadRequest, gin.H{ @@ -22,28 +23,46 @@ func Init(db interface{}, router *gin.Engine) { return } - assetStats, err := assetStat.Get(db, assetStat.Filter{From: from, To: to}) + assetStats, err := assetStat.Get(db, args) if err != nil { log.Printf("[ERROR] Couldn't get asset stats from database: %s", err) - c.String(http.StatusInternalServerError, "") + c.JSON(http.StatusInternalServerError, gin.H{ + "status": "error", + "message": "Internal server error", + }) return } c.JSON(http.StatusOK, gin.H{ - "stats": assetStats, + "status": "ok", + "stats": assetStats, }) return }) router.GET("/stats/latest", func(c *gin.Context) { - latest, err := assetStat.Latest(db) + args, err := filter.Parse(c) + if err != nil { + log.Printf("[ERROR] Couldn't parse URL parameters: %s", err) + c.JSON(http.StatusBadRequest, gin.H{ + "status": "error", + "message": err.Error(), + }) + return + } + + latest, err := assetStat.Latest(db, args) if err != nil { log.Printf("[ERROR] Couldn't get asset stats from database: %s", err) - c.String(http.StatusInternalServerError, "") + c.JSON(http.StatusInternalServerError, gin.H{ + "status": "error", + "message": "Internal server error", + }) return } c.JSON(http.StatusOK, gin.H{ + "status": "ok", "latest": latest, }) return @@ -51,6 +70,7 @@ func Init(db interface{}, router *gin.Engine) { router.GET("/stats/cursor", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{ + "status": "ok", "current_cursor": cursor.Current, }) return diff --git a/controllers/stats/stats_test.go b/controllers/stats/stats_test.go index b86831a..7b816ae 100644 --- a/controllers/stats/stats_test.go +++ b/controllers/stats/stats_test.go @@ -54,7 +54,7 @@ func TestStats(t *testing.T) { var tests = []HttpTestWithEffects{ { "GET", - "/stats", + fmt.Sprintf("/stats?asset_code=%s&asset_issuer=%s", cndy.AssetCode, cndy.AssetIssuer), "", http.StatusOK, test.Effects, @@ -63,7 +63,7 @@ func TestStats(t *testing.T) { // Filter{From} { "GET", - fmt.Sprintf("/stats?from=%s", test.Effects[4].CreatedAt.Format(time.RFC3339)), + fmt.Sprintf("/stats?asset_code=%s&asset_issuer=%s&from=%s", cndy.AssetCode, cndy.AssetIssuer, test.Effects[4].CreatedAt.Format(time.RFC3339)), "", http.StatusOK, test.Effects[4:], @@ -72,7 +72,7 @@ func TestStats(t *testing.T) { // Filter{To} { "GET", - fmt.Sprintf("/stats?to=%s", test.Effects[2].CreatedAt.Format(time.RFC3339)), + fmt.Sprintf("/stats?asset_code=%s&asset_issuer=%s&to=%s", cndy.AssetCode, cndy.AssetIssuer, test.Effects[2].CreatedAt.Format(time.RFC3339)), "", http.StatusOK, test.Effects[:3], @@ -81,11 +81,29 @@ func TestStats(t *testing.T) { // Filter{From, To} { "GET", - fmt.Sprintf("/stats?from=%s&to=%s", test.Effects[3].CreatedAt.Format(time.RFC3339), test.Effects[6].CreatedAt.Format(time.RFC3339)), + fmt.Sprintf("/stats?asset_code=%s&asset_issuer=%s&from=%s&to=%s", cndy.AssetCode, cndy.AssetIssuer, test.Effects[3].CreatedAt.Format(time.RFC3339), test.Effects[6].CreatedAt.Format(time.RFC3339)), "", http.StatusOK, test.Effects[3:7], }, + + // Invalid Filter{} + { + "GET", + fmt.Sprintf("/stats?asset_code=%s&asset_issuer=%s&from=xxx", cndy.AssetCode, cndy.AssetIssuer), + "", + http.StatusBadRequest, + nil, + }, + + // Missing asset_code and asset_issuer + { + "GET", + "/stats", + "", + http.StatusBadRequest, + nil, + }, } router := gin.Default() @@ -102,19 +120,32 @@ func TestStats(t *testing.T) { t.Errorf("Expected code %v, got %v, for %+v", tt.statusCode, resp.Code, tt) } - stats := make(map[string][]assetStat.AssetStat) - err := json.Unmarshal([]byte(resp.Body.String()), &stats) - if err != nil { - t.Error(err) + type resJson struct { + Status string + Stats []assetStat.AssetStat + } + + if tt.statusCode == http.StatusOK { + if !strings.Contains(resp.Body.String(), `"status":"ok"`) { + t.Errorf("Body did not contain ok status message: %s", resp.Body.String()) + } + } else { + if !strings.Contains(resp.Body.String(), `"status":"error"`) { + t.Errorf("Body did not contain error status message: %s", resp.Body.String()) + } + + // Skip to next test + continue } - _, ok := stats["stats"] - if !ok { - t.Error(`Expected element "stats" in JSON response`) + res := resJson{} + err := json.Unmarshal([]byte(resp.Body.String()), &res) + if err != nil { + t.Error(err) } - if len(stats["stats"]) != len(tt.expectedStats) { - t.Errorf("Expected %d JSON elements, got %d", len(tt.expectedStats), len(stats["stats"])) + if len(res.Stats) != len(tt.expectedStats) { + t.Errorf("Expected %d JSON elements, got %d", len(tt.expectedStats), len(res.Stats)) } for _, e := range tt.expectedStats { @@ -154,10 +185,11 @@ func TestLatestAndCursor(t *testing.T) { var tests = []HttpTest{ { "GET", - "/stats/latest", + fmt.Sprintf("/stats/latest?asset_code=%s&asset_issuer=%s", cndy.AssetCode, cndy.AssetIssuer), "", http.StatusOK, []string{ + `"status":"ok"`, fmt.Sprintf(`"paging_token":"%s"`, latestEffect.PagingToken), fmt.Sprintf(`"issued":"%s"`, bigint.ToString(latestEffect.Issued)), fmt.Sprintf(`"transferred":"%s"`, bigint.ToString(latestEffect.Transferred)), @@ -167,12 +199,22 @@ func TestLatestAndCursor(t *testing.T) { }, }, + // Missing asset_code and asset_issuer + { + "GET", + "/stats/latest", + "", + http.StatusBadRequest, + nil, + }, + { "GET", "/stats/cursor", "", http.StatusOK, []string{ + `"status":"ok"`, fmt.Sprintf(`"current_cursor":"%s"`, cndy.GenesisCursor), }, }, @@ -185,23 +227,41 @@ func TestLatestAndCursor(t *testing.T) { } Init(tx, router) - for _, test := range tests { - body := bytes.NewBufferString(test.body) - req, _ := http.NewRequest(test.method, test.url, body) + for _, tt := range tests { + body := bytes.NewBufferString(tt.body) + req, _ := http.NewRequest(tt.method, tt.url, body) resp := httptest.NewRecorder() router.ServeHTTP(resp, req) - if resp.Code != test.statusCode { - t.Errorf("Expected code %v, got %v, for %+v", test.statusCode, resp.Code, test) + if resp.Code != tt.statusCode { + t.Errorf("Expected code %v, got %v, for %+v", tt.statusCode, resp.Code, tt) + } + + if tt.statusCode == http.StatusOK { + if !strings.Contains(resp.Body.String(), `"status":"ok"`) { + t.Errorf("Body did not contain ok status message: %s", resp.Body.String()) + } + } else { + if !strings.Contains(resp.Body.String(), `"status":"error"`) { + t.Errorf("Body did not contain error status message: %s", resp.Body.String()) + } + + // Skip to next test + continue } - if len(test.bodyContains) > 0 { - for _, s := range test.bodyContains { + if len(tt.bodyContains) > 0 { + for _, s := range tt.bodyContains { if !strings.Contains(resp.Body.String(), s) { t.Errorf("Body did not contain '%s' in '%s'", s, resp.Body.String()) } } } + + // Check whether JSON ID is hidden (regression test) + if strings.Contains(resp.Body.String(), `"id":`) || strings.Contains(resp.Body.String(), `"Id":`) { + t.Errorf("Body did contain JSON ID (should be excluded) in '%s'", resp.Body.String()) + } } } diff --git a/db/migrations/0006_create_assets.down.sql b/db/migrations/0006_create_assets.down.sql new file mode 100644 index 0000000..a2988f5 --- /dev/null +++ b/db/migrations/0006_create_assets.down.sql @@ -0,0 +1 @@ +DROP TABLE assets; diff --git a/db/migrations/0006_create_assets.up.sql b/db/migrations/0006_create_assets.up.sql new file mode 100644 index 0000000..03ba8c2 --- /dev/null +++ b/db/migrations/0006_create_assets.up.sql @@ -0,0 +1,16 @@ +CREATE TABLE assets ( + -- Code and issuer combination needs to be unique, can serve as primary key + code character varying(12) not null, + issuer character varying(56) not null, + primary key (code, issuer), + + type character varying(64) default 'credit_alphanum4', + created_at timestamp without time zone default current_timestamp +); + +-- Register CNDY +INSERT INTO assets(type, code, issuer) VALUES( + 'credit_alphanum4', + 'CNDY', + 'GCJKC2MI63KSQ6MLE6GBSXPDKTDAK43WR522ZYR3F34NPM7Z5UEPIZNX' +); diff --git a/db/migrations/0007_asset_stats_procedure.down.sql b/db/migrations/0007_asset_stats_procedure.down.sql new file mode 100644 index 0000000..5001801 --- /dev/null +++ b/db/migrations/0007_asset_stats_procedure.down.sql @@ -0,0 +1,39 @@ +-- Recreate asset_stats table (taken from migration 0005) +CREATE TABLE asset_stats ( + id serial PRIMARY KEY, + paging_token character varying(64), + asset_type character varying(64), + asset_code character varying(12), + asset_issuer character varying(56), + issued bigint, + transferred bigint, + accounts_with_trustline integer, + accounts_with_payments integer, + payments integer, + created_at timestamp without time zone +); + +CREATE OR REPLACE FUNCTION repopulate_asset_stats() + RETURNS VOID +AS +$$ +DECLARE + t_row record; +BEGIN + TRUNCATE asset_stats; + FOR t_row in SELECT paging_token, effect_id, asset_type, asset_code, asset_issuer, created_at FROM effects ORDER BY effect_id LOOP + INSERT INTO asset_stats(paging_token, asset_code, asset_issuer, asset_type, created_at, issued, transferred, accounts_with_trustline, accounts_with_payments, payments) + VALUES (t_row.paging_token, t_row.asset_code, t_row.asset_issuer, t_row.asset_type, t_row.created_at, + (SELECT COALESCE(SUM(amount), 0) FROM effects WHERE type = 'account_debited' AND account = t_row.asset_issuer AND effect_id <= t_row.effect_id), + (SELECT COALESCE(SUM(amount), 0) FROM effects WHERE type = 'account_debited' AND account != t_row.asset_issuer AND effect_id <= t_row.effect_id), + (SELECT COUNT(DISTINCT account) FROM effects WHERE account != t_row.asset_issuer AND type = 'trustline_created' AND effect_id <= t_row.effect_id), + (SELECT COUNT(DISTINCT account) FROM effects WHERE account != t_row.asset_issuer AND type = 'account_debited' AND effect_id <= t_row.effect_id), + (SELECT COUNT(*) FROM effects WHERE type = 'account_debited' AND account != t_row.asset_issuer AND effect_id <= t_row.effect_id) + ); + END LOOP; +END; +$$ +LANGUAGE plpgsql; + +-- Re-populate asset_stats table +SELECT repopulate_asset_stats(); diff --git a/db/migrations/0007_asset_stats_procedure.up.sql b/db/migrations/0007_asset_stats_procedure.up.sql new file mode 100644 index 0000000..c2a3cbe --- /dev/null +++ b/db/migrations/0007_asset_stats_procedure.up.sql @@ -0,0 +1,44 @@ +CREATE OR REPLACE FUNCTION asset_stats(asset_code_filter character varying(12), asset_issuer_filter character varying(56)) + RETURNS TABLE ( + -- TODO: ID to order required? + effect_id character varying(56), + paging_token character varying(64), + asset_type character varying(64), + asset_code character varying(12), + asset_issuer character varying(56), + issued bigint, + transferred bigint, + accounts_with_trustline integer, + accounts_with_payments integer, + payments integer, + created_at timestamp without time zone + ) +AS +$$ +#variable_conflict use_column +DECLARE + t_row record; +BEGIN + FOR t_row in SELECT effect_id, paging_token, asset_type, asset_code, asset_issuer, created_at FROM effects ORDER BY created_at LOOP + -- Next if asset_code and asset_issuer do not match + CONTINUE WHEN t_row.asset_code != asset_code_filter AND t_row.asset_issuer != asset_issuer_filter; + + effect_id := t_row.effect_id; + paging_token := t_row.paging_token; + asset_type := t_row.asset_type; + asset_code := t_row.asset_code; + asset_issuer := t_row.asset_issuer; + created_at := t_row.created_at; + issued := (SELECT COALESCE(SUM(amount), 0) FROM effects WHERE type = 'account_debited' AND account = t_row.asset_issuer AND effect_id <= t_row.effect_id); + transferred := (SELECT COALESCE(SUM(amount), 0) FROM effects WHERE type = 'account_debited' AND account != t_row.asset_issuer AND effect_id <= t_row.effect_id); + accounts_with_trustline := (SELECT COUNT(DISTINCT account) FROM effects WHERE account != t_row.asset_issuer AND type = 'trustline_created' AND effect_id <= t_row.effect_id); + accounts_with_payments := (SELECT COUNT(DISTINCT account) FROM effects WHERE account != t_row.asset_issuer AND type = 'account_debited' AND effect_id <= t_row.effect_id); + payments := (SELECT COUNT(*) FROM effects WHERE type = 'account_debited' AND account != t_row.asset_issuer AND effect_id <= t_row.effect_id); + RETURN NEXT; + END LOOP; +END; +$$ +LANGUAGE plpgsql; + +DROP TABLE asset_stats; +DROP FUNCTION repopulate_asset_stats; diff --git a/main.go b/main.go index 7583fdc..7008fac 100644 --- a/main.go +++ b/main.go @@ -1,12 +1,13 @@ package main import ( + "github.com/cndy-store/analytics/controllers/assets" "github.com/cndy-store/analytics/controllers/docs" "github.com/cndy-store/analytics/controllers/effects" "github.com/cndy-store/analytics/controllers/stats" + "github.com/cndy-store/analytics/models/asset" "github.com/cndy-store/analytics/models/cursor" "github.com/cndy-store/analytics/models/effect" - "github.com/cndy-store/analytics/utils/cndy" "github.com/cndy-store/analytics/utils/sql" "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" @@ -63,18 +64,28 @@ func main() { os.Exit(1) } + // Load registered assets into asset.Registered + asset.UpdateRegistered(db) + if err != nil { + log.Printf("[ERROR] Couldn't get registered assets from database: %s", err) + os.Exit(1) + } + for { client.StreamEffects(ctx, &cursor.Current, func(e horizon.Effect) { - if e.Asset.Code == cndy.AssetCode && e.Asset.Issuer == cndy.AssetIssuer { - err = effect.New(db, e) - if err != nil { - log.Printf("[ERROR] Couldn't save effect to database: %s", err) - } - - // Make sure to also safe the current cursor, so database is consistent - err = cursor.Save(db) - if err != nil { - log.Printf("[ERROR] Couldn't save cursor to database: %s", err) + // Check whether this asset was registered + for _, registeredAsset := range asset.Registered { + if e.Asset.Code == *registeredAsset.Code && e.Asset.Issuer == *registeredAsset.Issuer { + err = effect.New(db, e) + if err != nil { + log.Printf("[ERROR] Couldn't save effect to database: %s", err) + } + + // Make sure to also safe the current cursor, so database is consistent + err = cursor.Save(db) + if err != nil { + log.Printf("[ERROR] Couldn't save cursor to database: %s", err) + } } } @@ -87,8 +98,9 @@ func api(db *sqlx.DB) { router := gin.Default() router.Use(cors.Default()) // Allow all origins - stats.Init(db, router) + assets.Init(db, router) effects.Init(db, router) + stats.Init(db, router) docs.Init(router) router.Run(":3144") diff --git a/models/asset/asset.go b/models/asset/asset.go new file mode 100644 index 0000000..a3f62bf --- /dev/null +++ b/models/asset/asset.go @@ -0,0 +1,49 @@ +package asset + +import ( + "github.com/cndy-store/analytics/utils/sql" + "time" +) + +type Asset struct { + Type *string `db:"type" json:"type,omitempty"` + Code *string `db:"code" json:"code,omitempty" binding="required"` + Issuer *string `db:"issuer" json:"issuer,omitempty binding="required"` + CreatedAt *time.Time `db:"created_at" json:"created_at,omitempty"` +} + +// Variable to hold all registered assets in memory +var Registered []Asset + +func New(db sql.Database, asset Asset) (ret Asset, err error) { + err = db.Get(&ret, `INSERT INTO assets(type, code, issuer) VALUES($1, $2, $3) RETURNING *`, + asset.Type, asset.Code, asset.Issuer) + if err != nil { + return + } + + err = UpdateRegistered(db) + return +} + +func Exists(db sql.Database, asset Asset) (yes bool, err error) { + return sql.Exists(db, `SELECT 1 FROM assets WHERE code=$1 AND issuer=$2`, asset.Code, asset.Issuer) +} + +func Get(db sql.Database) (assets []Asset, err error) { + err = db.Select(&assets, `SELECT * FROM assets`) + if err == sql.ErrNoRows { + err = nil + return + } + if err != nil { + return + } + + return +} + +func UpdateRegistered(db sql.Database) (err error) { + Registered, err = Get(db) + return +} diff --git a/models/asset_stat/asset_stat.go b/models/asset_stat/asset_stat.go index abc41cc..a63d9a1 100644 --- a/models/asset_stat/asset_stat.go +++ b/models/asset_stat/asset_stat.go @@ -2,14 +2,13 @@ package assetStat import ( "github.com/cndy-store/analytics/utils/bigint" + "github.com/cndy-store/analytics/utils/filter" "github.com/cndy-store/analytics/utils/sql" - "github.com/stellar/go/clients/horizon" - "log" "time" ) type AssetStat struct { - Id *uint32 `db:"id", json:"-"` + EffectId *string `db:"effect_id", json:"effect_id"` // TODO: omitempty, also add to tests? PagingToken *string `db:"paging_token" json:"paging_token,omitempty"` AssetType *string `db:"asset_type" json:"asset_type,omitempty"` AssetCode *string `db:"asset_code" json:"asset_code,omitempty"` @@ -29,23 +28,6 @@ type AssetStat struct { JsonTransferred *string `db:"-" json:"transferred"` } -func New(db interface{}, effect horizon.Effect, timestamp time.Time) (err error) { - // Store amount_transfered and amount_issued upon insert in a different table - // (analogue to the asset endpoint of Horizon) - - _, err = sql.Exec(db, `INSERT INTO asset_stats(paging_token, asset_code, asset_issuer, asset_type, created_at, issued, transferred, accounts_with_trustline, accounts_with_payments, payments) - VALUES ($1, $2, $3, $4, $5, - (SELECT COALESCE(SUM(amount), 0) FROM effects WHERE type='account_debited' AND account=$6), - (SELECT COALESCE(SUM(amount), 0) FROM effects WHERE type='account_debited' AND account!=$6), - (SELECT COUNT(DISTINCT account) FROM effects WHERE type='trustline_created' AND account!=$6), - (SELECT COUNT(DISTINCT account) FROM effects WHERE type='account_debited' AND account!=$6), - (SELECT COUNT(*) FROM effects WHERE type='account_debited' AND account!=$6) - )`, - effect.PT, effect.Asset.Code, effect.Asset.Issuer, effect.Asset.Type, timestamp, effect.Asset.Issuer) - - return -} - type Filter struct { From *time.Time To *time.Time @@ -63,12 +45,16 @@ func (f *Filter) Defaults() { } } -func Get(db interface{}, filter Filter) (stats []AssetStat, err error) { +func Get(db sql.Database, filter filter.Filter) (stats []AssetStat, err error) { filter.Defaults() - err = sql.Select(db, &stats, `SELECT * FROM asset_stats WHERE created_at BETWEEN $1::timestamp AND $2::timestamp ORDER BY id`, - filter.From, filter.To) + err = db.Select(&stats, `SELECT * FROM asset_stats($1, $2) WHERE created_at BETWEEN $3::timestamp AND $4::timestamp ORDER BY effect_id`, + filter.AssetCode, filter.AssetIssuer, filter.From, filter.To) if err == sql.ErrNoRows { - log.Printf("[ERROR] asset_stat.Get(): %s", err) + err = nil + return + } + if err != nil { + return } // Convert int64 fields to strings @@ -79,10 +65,16 @@ func Get(db interface{}, filter Filter) (stats []AssetStat, err error) { return } -func Latest(db interface{}) (stats AssetStat, err error) { - err = sql.Get(db, &stats, `SELECT * FROM asset_stats ORDER BY id DESC LIMIT 1`) +func Latest(db sql.Database, filter filter.Filter) (stats AssetStat, err error) { + filter.Defaults() + err = db.Get(&stats, `SELECT * FROM asset_stats($1, $2) ORDER BY effect_id DESC LIMIT 1`, + filter.AssetCode, filter.AssetIssuer) if err == sql.ErrNoRows { - log.Printf("[ERROR] asset_stat.Latest(): %s", err) + err = nil + return + } + if err != nil { + return } // Convert int64 fields to strings diff --git a/models/asset_stat/asset_stat_test.go b/models/asset_stat/asset_stat_test.go index e087071..51434ae 100644 --- a/models/asset_stat/asset_stat_test.go +++ b/models/asset_stat/asset_stat_test.go @@ -1,6 +1,7 @@ package assetStat import ( + "github.com/cndy-store/analytics/utils/filter" "github.com/cndy-store/analytics/utils/sql" "github.com/cndy-store/analytics/utils/test" "testing" @@ -24,7 +25,7 @@ func TestGet(t *testing.T) { } // Filter{} - assetStats, err := Get(tx, Filter{}) + assetStats, err := Get(tx, filter.NewCNDYFilter(nil, nil)) if err != nil { t.Errorf("assetStat.Get(): %s", err) } @@ -33,7 +34,7 @@ func TestGet(t *testing.T) { } // Filter{From} - assetStats, err = Get(tx, Filter{From: &test.Effects[2].CreatedAt}) + assetStats, err = Get(tx, filter.NewCNDYFilter(&test.Effects[2].CreatedAt, nil)) if err != nil { t.Errorf("assetStat.Get(): %s", err) } @@ -64,7 +65,7 @@ func TestGet(t *testing.T) { } // Filter{To} - assetStats, err = Get(tx, Filter{To: &test.Effects[1].CreatedAt}) + assetStats, err = Get(tx, filter.NewCNDYFilter(nil, &test.Effects[1].CreatedAt)) if err != nil { t.Errorf("assetStat.Get(): %s", err) } @@ -95,7 +96,7 @@ func TestGet(t *testing.T) { } // Filter{From, To} - assetStats, err = Get(tx, Filter{From: &test.Effects[1].CreatedAt, To: &test.Effects[2].CreatedAt}) + assetStats, err = Get(tx, filter.NewCNDYFilter(&test.Effects[1].CreatedAt, &test.Effects[2].CreatedAt)) if err != nil { t.Errorf("assetStat.Get(): %s", err) } @@ -144,7 +145,7 @@ func TestLatest(t *testing.T) { } // Filter{} - assetStats, err := Latest(tx) + assetStats, err := Latest(tx, filter.NewCNDYFilter(nil, nil)) if err != nil { t.Errorf("assetStat.Latest(): %s", err) } diff --git a/models/cursor/cursor.go b/models/cursor/cursor.go index 11577f9..9a9c464 100644 --- a/models/cursor/cursor.go +++ b/models/cursor/cursor.go @@ -18,14 +18,14 @@ func Update(cursor horizon.Cursor) { Current = cursor } -func Save(db interface{}) (err error) { - _, err = sql.Exec(db, `UPDATE cursors SET paging_token=$1 WHERE id=1`, Current) +func Save(db sql.Database) (err error) { + _, err = db.Exec(`UPDATE cursors SET paging_token=$1 WHERE id=1`, Current) return } -func LoadLatest(db interface{}) (err error) { +func LoadLatest(db sql.Database) (err error) { var c string - err = sql.Get(db, &c, `SELECT paging_token FROM cursors WHERE id=1`) + err = db.Get(&c, `SELECT paging_token FROM cursors WHERE id=1`) if err != nil { return } diff --git a/models/effect/effect.go b/models/effect/effect.go index ccffbb8..8689820 100644 --- a/models/effect/effect.go +++ b/models/effect/effect.go @@ -2,8 +2,8 @@ package effect import ( "encoding/json" - "github.com/cndy-store/analytics/models/asset_stat" "github.com/cndy-store/analytics/utils/bigint" + "github.com/cndy-store/analytics/utils/filter" "github.com/cndy-store/analytics/utils/sql" "github.com/stellar/go/clients/horizon" "log" @@ -49,7 +49,7 @@ type Operation struct { CreatedAt time.Time `json:"created_at"` } -func New(db interface{}, effect horizon.Effect) (err error) { +func New(db sql.Database, effect horizon.Effect) (err error) { // Get operation operation := getOperation(effect.Links.Operation.Href) @@ -79,8 +79,7 @@ func New(db interface{}, effect horizon.Effect) (err error) { return } - // Just input the fields we're requiring for now, can be replayed anytime form the chain later. - _, err = sql.Exec(db, `INSERT INTO effects( + _, err = db.Exec(`INSERT INTO effects( effect_id, operation, succeeds, precedes, paging_token, account, amount, type, type_i, starting_balance, @@ -109,12 +108,6 @@ func New(db interface{}, effect horizon.Effect) (err error) { return } - // Store asset stats upon insert in a different table - err = assetStat.New(db, effect, operation.CreatedAt) - if err != nil { - return - } - log.Printf("--+--[ %s ]", effect.Asset.Code) log.Printf(" |") log.Printf(" +-> Type: %s", effect.Type) @@ -124,30 +117,16 @@ func New(db interface{}, effect horizon.Effect) (err error) { return } -type Filter struct { - Type string - From *time.Time - To *time.Time -} - -func (f *Filter) Defaults() { - if f.From == nil { - t := time.Unix(0, 0) - f.From = &t - } - - if f.To == nil { - t := time.Now() - f.To = &t - } -} - -func Get(db interface{}, filter Filter) (effects []Effect, err error) { +func Get(db sql.Database, filter filter.Filter) (effects []Effect, err error) { filter.Defaults() - err = sql.Select(db, &effects, `SELECT * FROM effects WHERE created_at BETWEEN $1::timestamp AND $2::timestamp ORDER BY created_at`, - filter.From, filter.To) + err = db.Select(&effects, `SELECT * FROM effects WHERE asset_code=$1 AND asset_issuer=$2 AND created_at BETWEEN $3::timestamp AND $4::timestamp ORDER BY created_at`, + filter.AssetCode, filter.AssetIssuer, filter.From, filter.To) if err == sql.ErrNoRows { - log.Printf("[ERROR] effect.Get(): %s", err) + err = nil + return + } + if err != nil { + return } // Convert int64 fields to strings diff --git a/models/effect/effect_test.go b/models/effect/effect_test.go index 4129dce..01d6933 100644 --- a/models/effect/effect_test.go +++ b/models/effect/effect_test.go @@ -4,6 +4,7 @@ import ( "github.com/cndy-store/analytics/models/asset_stat" "github.com/cndy-store/analytics/utils/bigint" "github.com/cndy-store/analytics/utils/cndy" + "github.com/cndy-store/analytics/utils/filter" "github.com/cndy-store/analytics/utils/sql" "github.com/cndy-store/analytics/utils/test" "github.com/stellar/go/clients/horizon" @@ -70,7 +71,7 @@ func TestNew(t *testing.T) { t.Error(err) } - effects, err := Get(tx, Filter{}) + effects, err := Get(tx, filter.NewCNDYFilter(nil, nil)) if err != nil { t.Error(err) } @@ -130,7 +131,7 @@ func TestNew(t *testing.T) { } // Check whether asset_stat data was updated - a, err := assetStat.Latest(tx) + a, err := assetStat.Latest(tx, filter.NewCNDYFilter(nil, nil)) if err != nil { t.Error(err) } @@ -175,8 +176,8 @@ func TestGet(t *testing.T) { t.Error(err) } - // Filter{} - effects, err := Get(tx, Filter{}) + // NewCNDYFilter(nil, nil) + effects, err := Get(tx, filter.NewCNDYFilter(nil, nil)) if err != nil { t.Errorf("effect.Get(): %s", err) } @@ -185,7 +186,7 @@ func TestGet(t *testing.T) { } // Filter{From} - effects, err = Get(tx, Filter{From: &test.Effects[5].CreatedAt}) + effects, err = Get(tx, filter.NewCNDYFilter(&test.Effects[5].CreatedAt, nil)) if err != nil { t.Errorf("effect.Get(): %s", err) } @@ -201,7 +202,7 @@ func TestGet(t *testing.T) { } // Filter{To} - effects, err = Get(tx, Filter{To: &test.Effects[2].CreatedAt}) + effects, err = Get(tx, filter.NewCNDYFilter(nil, &test.Effects[2].CreatedAt)) if err != nil { t.Errorf("effect.Get(): %s", err) } @@ -217,7 +218,7 @@ func TestGet(t *testing.T) { } // Filter{From, To} - effects, err = Get(tx, Filter{From: &test.Effects[3].CreatedAt, To: &test.Effects[4].CreatedAt}) + effects, err = Get(tx, filter.NewCNDYFilter(&test.Effects[3].CreatedAt, &test.Effects[4].CreatedAt)) if err != nil { t.Errorf("effect.Get(): %s", err) } @@ -226,7 +227,6 @@ func TestGet(t *testing.T) { } for i, e := range test.Effects[3:5] { - if e.PagingToken != *effects[i].PagingToken { t.Errorf("Expected paging_token to be %s got: %s", e.PagingToken, *effects[i].PagingToken) } diff --git a/utils/filter/filter.go b/utils/filter/filter.go index a6737e9..ac945b2 100644 --- a/utils/filter/filter.go +++ b/utils/filter/filter.go @@ -2,18 +2,26 @@ package filter import ( "errors" + "github.com/cndy-store/analytics/utils/cndy" "github.com/gin-gonic/gin" "time" ) -func Parse(c *gin.Context) (from *time.Time, to *time.Time, err error) { +type Filter struct { + From *time.Time + To *time.Time + AssetCode string + AssetIssuer string +} + +func Parse(c *gin.Context) (filter Filter, err error) { if query := c.Query("from"); query != "" { t, e := time.Parse(time.RFC3339, query) if e != nil { err = errors.New("Invalid date in 'from' parameter.") return } - from = &t + filter.From = &t } if query := c.Query("to"); query != "" { @@ -22,8 +30,55 @@ func Parse(c *gin.Context) (from *time.Time, to *time.Time, err error) { err = errors.New("Invalid date in 'to' parameter.") return } - to = &t + filter.To = &t + } + + if query := c.Query("asset_code"); query != "" { + filter.AssetCode = query + } else { + err = errors.New("Missing 'asset_code' parameter") + return + } + + if query := c.Query("asset_issuer"); query != "" { + filter.AssetIssuer = query + } else { + err = errors.New("Missing 'asset_issuer' parameter") + return } return } + +func (f *Filter) Defaults() { + if f.From == nil { + t := time.Unix(0, 0) + f.From = &t + } + + if f.To == nil { + t := time.Now() + f.To = &t + } +} + +// Returns a filter object with pre-filled AssetIssuer and AssetCode +// for the CNDY coin used in testing +func NewCNDYFilter(from *time.Time, to *time.Time) Filter { + if from == nil { + t := time.Unix(0, 0) + from = &t + } + + if to == nil { + t := time.Now() + to = &t + } + + return Filter{ + From: from, + To: to, + AssetCode: cndy.AssetCode, + AssetIssuer: cndy.AssetIssuer, + } +} diff --git a/utils/sql/sql.go b/utils/sql/sql.go index 4dbebd9..5905202 100644 --- a/utils/sql/sql.go +++ b/utils/sql/sql.go @@ -1,8 +1,8 @@ package sql import ( + "context" "database/sql" - "errors" "github.com/golang-migrate/migrate" "github.com/golang-migrate/migrate/database/postgres" _ "github.com/golang-migrate/migrate/source/file" @@ -12,6 +12,42 @@ import ( var ErrNoRows = sql.ErrNoRows +// This type serves as an abstraction for sqlx.DB and sqlx.Tx and supports all functions both +// of the types have in common. This allows as to use their functions type agnostically. +type Database interface { + // Common functions of sql.DB and sql.Tx + Exec(query string, args ...interface{}) (sql.Result, error) + ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) + Prepare(query string) (*sql.Stmt, error) + PrepareContext(ctx context.Context, query string) (*sql.Stmt, error) + Query(query string, args ...interface{}) (*sql.Rows, error) + QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) + QueryRow(query string, args ...interface{}) *sql.Row + QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row + + // Common functions of sqlx.DB and sqlx.Tx + BindNamed(query string, arg interface{}) (string, []interface{}, error) + DriverName() string + Get(dest interface{}, query string, args ...interface{}) error + GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error + MustExec(query string, args ...interface{}) sql.Result + MustExecContext(ctx context.Context, query string, args ...interface{}) sql.Result + NamedExec(query string, arg interface{}) (sql.Result, error) + NamedExecContext(ctx context.Context, query string, arg interface{}) (sql.Result, error) + NamedQuery(query string, arg interface{}) (*sqlx.Rows, error) + PrepareNamed(query string) (*sqlx.NamedStmt, error) + PrepareNamedContext(ctx context.Context, query string) (*sqlx.NamedStmt, error) + Preparex(query string) (*sqlx.Stmt, error) + PreparexContext(ctx context.Context, query string) (*sqlx.Stmt, error) + QueryRowx(query string, args ...interface{}) *sqlx.Row + QueryRowxContext(ctx context.Context, query string, args ...interface{}) *sqlx.Row + Queryx(query string, args ...interface{}) (*sqlx.Rows, error) + QueryxContext(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) + Rebind(query string) string + Select(dest interface{}, query string, args ...interface{}) error + SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error +} + // Open production database and run migrations func OpenAndMigrate(relPath string) (db *sqlx.DB, err error) { /* Open connection to postgresql to the server specified in the following environment variables: @@ -44,57 +80,10 @@ func OpenAndMigrate(relPath string) (db *sqlx.DB, err error) { return } -// Exec is a type agnostic wrapper for sqlx.Exec() (works with sqlx.DB and sqlx.Tx) -func Exec(db interface{}, query string, args ...interface{}) (result sql.Result, err error) { - switch db.(type) { - case *sqlx.DB: - result, err = db.(*sqlx.DB).Exec(query, args...) - case *sqlx.Tx: - result, err = db.(*sqlx.Tx).Exec(query, args...) - default: - err = errors.New("Unknown DB interface{} in sql.Exec()") - } - return -} - -// Getis a type agnostic wrapper for sqlx.Get() (works with sqlx.DB and sqlx.Tx) -func Get(db, obj interface{}, query string, args ...interface{}) (err error) { - switch db.(type) { - case *sqlx.DB: - err = db.(*sqlx.DB).Get(obj, query, args...) - case *sqlx.Tx: - err = db.(*sqlx.Tx).Get(obj, query, args...) - default: - err = errors.New("Unknown DB interface{} in sql.Get()") - } - return -} - -// Select is a type agnostic wrapper for sqlx.Select() (works with sqlx.DB and sqlx.Tx) -func Select(db interface{}, obj interface{}, query string, args ...interface{}) (err error) { - switch db.(type) { - case *sqlx.DB: - err = db.(*sqlx.DB).Select(obj, query, args...) - case *sqlx.Tx: - err = db.(*sqlx.Tx).Select(obj, query, args...) - default: - err = errors.New("Unknown DB interface{} in sql.Select()") - } - return -} - // NamedQuery is a type agnostic wrapper for sqlx.NamedQuery() (works with sqlx.DB and sqlx.Tx) -func NamedQuery(db interface{}, obj interface{}, query string, arg interface{}) (err error) { +func NamedQuery(db Database, obj interface{}, query string, arg interface{}) (err error) { var stmt *sqlx.NamedStmt - - switch db.(type) { - case *sqlx.DB: - stmt, err = db.(*sqlx.DB).PrepareNamed(query) - case *sqlx.Tx: - stmt, err = db.(*sqlx.Tx).PrepareNamed(query) - default: - err = errors.New("Unknown DB interface{} in sql.NamedQuery()") - } + stmt, err = db.PrepareNamed(query) if err != nil { return @@ -110,20 +99,12 @@ func NamedQuery(db interface{}, obj interface{}, query string, arg interface{}) } // Exists is a type agnostic function that checks whether a statement returns a row -func Exists(db interface{}, query string, args ...interface{}) (exists bool, err error) { +func Exists(db Database, query string, args ...interface{}) (exists bool, err error) { var row *sql.Row // Prepare exists query query = `SELECT EXISTS(` + query + `) LIMIT 1` - - switch db.(type) { - case *sqlx.DB: - row = db.(*sqlx.DB).QueryRow(query, args...) - case *sqlx.Tx: - row = db.(*sqlx.Tx).QueryRow(query, args...) - default: - err = errors.New("Unknown DB interface{} in sql.Exists()") - } + row = db.QueryRow(query, args...) err = row.Scan(&exists) return diff --git a/utils/test/test.go b/utils/test/test.go index 7c98796..96c070a 100644 --- a/utils/test/test.go +++ b/utils/test/test.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/cndy-store/analytics/utils/bigint" "github.com/cndy-store/analytics/utils/cndy" - "github.com/cndy-store/analytics/utils/sql" "github.com/jmoiron/sqlx" "time" ) @@ -50,7 +49,7 @@ func InsertTestData(tx *sqlx.Tx) (err error) { return } - _, err = sql.Exec(tx, `INSERT INTO effects(effect_id, operation, paging_token, account, amount, type, asset_type, asset_issuer, asset_code, created_at) + _, err = tx.Exec(`INSERT INTO effects(effect_id, operation, paging_token, account, amount, type, asset_type, asset_issuer, asset_code, created_at) VALUES($1, 'https://horizon-testnet.stellar.org/operations/34028708058632193', $2, $3, $4, $5, 'credit_alphanum4', $6, $7, $8)`, fmt.Sprintf("0034028708058632193-%09d", i), data.PagingToken, data.Account, amount, data.Type, cndy.AssetIssuer, cndy.AssetCode, data.CreatedAt) if err != nil { @@ -58,10 +57,5 @@ func InsertTestData(tx *sqlx.Tx) (err error) { } } - _, err = sql.Exec(tx, `SELECT repopulate_asset_stats()`) - if err != nil { - return - } - return }