Merge pull request #464 from AlisProject/add_bcg_ranking

Add search tags count.

Author: keillera

function02-template.yaml

@@ -311,6 +311,23 @@ Resources:
       Timeout: 15
       TracingConfig:
         Mode: "Active"
+  SearchTagsCount:
+    Type: "AWS::Lambda::Function"
+    Properties:
+      Code: ./deploy/search_tags_count.zip
+      Environment:
+        Variables:
+          ELASTIC_SEARCH_ENDPOINT: !Ref ElasticSearchEndpoint
+      Handler: handler.lambda_handler
+      MemorySize: 3008
+      Role:
+        Fn::ImportValue:
+          Fn::Sub: "${AlisAppId}-LambdaRole"
+      Runtime: python3.6
+      Timeout: 300
+      TracingConfig:
+        Mode: "Active"
+
 
 Outputs:
   ArticleSupportersIndex:
@@ -373,3 +390,7 @@ Outputs:
     Value: !GetAtt TopicsCryptoRankingIndex.Arn
     Export:
       Name: !Sub "${AlisAppId}-TopicsCryptoRankingIndex"
+  SearchTagsCount:
+    Value: !GetAtt SearchTagsCount.Arn
+    Export:
+      Name: !Sub "${AlisAppId}-SearchTagsCount"

permission-template.yaml

@@ -323,3 +323,12 @@ Resources:
           Fn::Sub: "${AlisAppId}-TopicsCryptoRankingIndex"
       Principal: "apigateway.amazonaws.com"
       SourceArn: !Sub ${RestApiArn}/*/GET/topics/crypto/ranking
+  SearchTagsCountApiGatewayInvoke:
+    Type: "AWS::Lambda::Permission"
+    Properties:
+      Action: "lambda:InvokeFunction"
+      FunctionName:
+        Fn::ImportValue:
+          Fn::Sub: "${AlisAppId}-SearchTagsCount"
+      Principal: "apigateway.amazonaws.com"
+      SourceArn: !Sub ${RestApiArn}/*/GET/search/tags_count

src/common/es_util.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import settings
 import os
+import time
 
 
 class ESUtil:
@@ -38,6 +39,35 @@ def search_tag(elasticsearch, word, limit, page):
 
         return tags
 
+    @staticmethod
+    def search_tags_count(elasticsearch, size=500, term=604800):
+        from_time = round(time.time()) - term
+        body = {
+          'size': 0,
+          'query': {
+            'bool': {
+              'must': [
+                {'range': {'published_at': {'gte': from_time}}}
+              ]
+            }
+          },
+          'aggs': {
+            'tags_count': {
+              'terms': {
+                'field': 'tags.keyword',
+                'order': {'_count': 'desc'},
+                'size': size
+              }
+            }
+          }
+        }
+
+        response = elasticsearch.search(
+            index='articles',
+            body=body
+        )
+        return response['aggregations']['tags_count']['buckets']
+
     @staticmethod
     def search_article(elasticsearch, limit, page, word=None, tag=None):
         body = {

src/common/settings.py

@@ -150,6 +150,15 @@
         },
         'maxItems': 5
     },
+    'tags_count': {
+        'type': 'array',
+        'items': {
+            'type': 'string',
+            'minLength': 1,
+            'maxLength': 25
+        },
+        'maxItems': 150
+    },
     'tip_value': {
         'type': 'number',
         'minimum': 1,
@@ -274,7 +283,12 @@
     },
     'pin_code': {
         'type': 'string'
-    }
+    },
+    'tags_count_search_days': {
+        'type': 'integer',
+        'minimum': 1,
+        'maximum': 10000
+    },
 }
 
 # ログに出力されてはいけないパラメータ(ログ出力時に値がマスクされる)

src/handlers/search/tags_count/handler.py

@@ -0,0 +1,24 @@
+import os
+from search_tags_count import SearchTagsCount
+from elasticsearch import Elasticsearch, RequestsHttpConnection
+from requests_aws4auth import AWS4Auth
+
+awsauth = AWS4Auth(
+    os.environ['AWS_ACCESS_KEY_ID'],
+    os.environ['AWS_SECRET_ACCESS_KEY'],
+    os.environ['AWS_REGION'],
+    'es',
+    session_token=os.environ['AWS_SESSION_TOKEN']
+)
+elasticsearch = Elasticsearch(
+    hosts=[{'host': os.environ['ELASTIC_SEARCH_ENDPOINT'], 'port': 443}],
+    http_auth=awsauth,
+    use_ssl=True,
+    verify_certs=True,
+    connection_class=RequestsHttpConnection
+)
+
+
+def lambda_handler(event, context):
+    search_tags_count = SearchTagsCount(event, context, elasticsearch=elasticsearch)
+    return search_tags_count.main()

src/handlers/search/tags_count/search_tags_count.py

@@ -0,0 +1,47 @@
+import json
+
+from jsonschema import validate
+
+import settings
+from decimal_encoder import DecimalEncoder
+from es_util import ESUtil
+from lambda_base import LambdaBase
+from parameter_util import ParameterUtil
+
+
+class SearchTagsCount(LambdaBase):
+    def get_schema(self):
+        return {
+            'type': 'object',
+            'properties': {
+                'tags': settings.parameters['tags_count'],
+                'search_days': settings.parameters['tags_count_search_days']
+            },
+            'required': ['tags']
+        }
+
+    def validate_params(self):
+        ParameterUtil.cast_parameter_to_int(self.params, self.get_schema())
+        self.params['tags'] = self.event['multiValueQueryStringParameters'].get('tags')
+        validate(self.params, self.get_schema())
+
+    def exec_main_proc(self):
+        # 直近１週間分のタグを集計
+        search_size = len(self.params['tags']) * settings.parameters['tags']['maxItems']
+        search_days = self.params.get('search_days') if self.params.get('search_days') else 7
+        from_time = 86400 * search_days
+        search_result = ESUtil.search_tags_count(self.elasticsearch, search_size, from_time)
+
+        # 集計結果より指定タグの件数を取得
+        temp = []
+        for tag in self.params['tags']:
+            tag_count = [d['doc_count'] for d in search_result if d['key'] == tag]
+            temp.append({
+                'tag': tag,
+                'count': tag_count[0] if len(tag_count) > 0 else 0
+            })
+        result = sorted(temp, key=lambda x: x['count'], reverse=True)
+        return {
+            'statusCode': 200,
+            'body': json.dumps(result, cls=DecimalEncoder)
+        }

swagger/swagger.yaml

@@ -256,6 +256,13 @@ definitions:
         type: integer
       created_at:
         type: integer
+  TagCount:
+    type: object
+    properties:
+      tag:
+        type: string
+      count:
+        type: integer
   Fraud:
     type: object
     properties:
@@ -498,6 +505,43 @@ paths:
         passthroughBehavior: when_no_templates
         httpMethod: POST
         type: aws_proxy
+  /search/tags_count:
+    get:
+      description: "タグ件数検索"
+      parameters:
+      - name: "tags"
+        in: "query"
+        description: "検索タグ"
+        required: true
+        type: array
+        items:
+          type: string
+      - name: "search_days"
+        in: "query"
+        description: "検索対象の日数"
+        required: false
+        type: "integer"
+      responses:
+        "200":
+          description: "タグ件数一覧"
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/TagCount'
+      x-amazon-apigateway-integration:
+        responses:
+          default:
+            statusCode: "200"
+        uri:
+          Fn::Join:
+            - ''
+            - - Fn::Sub: "arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/"
+              - Fn::ImportValue:
+                  Fn::Sub: "${AlisAppId}-SearchTagsCount"
+              - "/invocations"
+        passthroughBehavior: when_no_templates
+        httpMethod: POST
+        type: aws_proxy
   /articles/recent:
     get:
       description: "最新記事一覧情報を取得"